You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by jh...@apache.org on 2015/07/20 10:34:19 UTC
[01/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Repository: tajo
Updated Branches:
refs/heads/master bedce3aa0 -> 5a02873d5
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskExecutor.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskExecutor.java
index f62733f..2340551 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskExecutor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskExecutor.java
@@ -18,33 +18,34 @@
package org.apache.tajo.worker;
-import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
+import org.apache.tajo.ResourceProtos.TaskStatusProto;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.worker.event.TaskExecutorEvent;
+import org.apache.tajo.worker.event.TaskStartEvent;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.Semaphore;
public class MockTaskExecutor extends TaskExecutor {
protected final Semaphore barrier;
- public MockTaskExecutor(Semaphore barrier, TaskManager taskManager, EventHandler rmEventHandler) {
- super(taskManager, rmEventHandler);
+ public MockTaskExecutor(Semaphore barrier, TajoWorker.WorkerContext workerContext) {
+ super(workerContext);
this.barrier = barrier;
}
@Override
- public void handle(TaskExecutorEvent event) {
+ public void handle(TaskStartEvent event) {
super.handle(event);
barrier.release();
}
@Override
- protected Task createTask(final ExecutionBlockContext context, TajoWorkerProtocol.TaskRequestProto taskRequest) {
+ protected Task createTask(final ExecutionBlockContext context, TaskRequestProto taskRequest) {
final TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
//ignore status changed log
@@ -126,8 +127,8 @@ public class MockTaskExecutor extends TaskExecutor {
}
@Override
- public TajoWorkerProtocol.TaskStatusProto getReport() {
- TajoWorkerProtocol.TaskStatusProto.Builder builder = TajoWorkerProtocol.TaskStatusProto.newBuilder();
+ public TaskStatusProto getReport() {
+ TaskStatusProto.Builder builder = TaskStatusProto.newBuilder();
builder.setWorkerName("localhost:0");
builder.setId(taskAttemptContext.getTaskId().getProto())
.setProgress(taskAttemptContext.getProgress())
@@ -136,6 +137,16 @@ public class MockTaskExecutor extends TaskExecutor {
builder.setInputStats(new TableStats().getProto());
return builder.build();
}
+
+ @Override
+ public TaskHistory createTaskHistory() {
+ return null;
+ }
+
+ @Override
+ public List<Fetcher> getFetchers() {
+ return null;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskManager.java
index 678b063..76ce9f7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskManager.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockTaskManager.java
@@ -19,9 +19,13 @@
package org.apache.tajo.worker;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse;
+import org.apache.tajo.ResourceProtos.ExecutionBlockListProto;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.worker.event.TaskManagerEvent;
import java.io.IOException;
@@ -31,15 +35,21 @@ public class MockTaskManager extends TaskManager {
private final Semaphore barrier;
- public MockTaskManager(Semaphore barrier, Dispatcher dispatcher, TajoWorker.WorkerContext workerContext, EventHandler rmEventHandler) {
- super(dispatcher, workerContext, rmEventHandler);
+ public MockTaskManager(Semaphore barrier, Dispatcher dispatcher, TajoWorker.WorkerContext workerContext) {
+ super(dispatcher, workerContext);
this.barrier = barrier;
}
@Override
- protected ExecutionBlockContext createExecutionBlock(TajoWorkerProtocol.RunExecutionBlockRequestProto request) {
+ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionBlockId, String queryMaster) {
try {
- return new MockExecutionBlock(getWorkerContext(), request);
+ ExecutionBlockContextResponse.Builder builder = ExecutionBlockContextResponse.newBuilder();
+ builder.setExecutionBlockId(executionBlockId.getProto())
+ .setPlanJson("test")
+ .setQueryContext(new QueryContext(new TajoConf()).getProto())
+ .setQueryOutputPath("testpath")
+ .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
+ return new MockExecutionBlock(getWorkerContext(), builder.build());
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -47,7 +57,7 @@ public class MockTaskManager extends TaskManager {
@Override
protected void stopExecutionBlock(ExecutionBlockContext context,
- TajoWorkerProtocol.ExecutionBlockListProto cleanupList) {
+ ExecutionBlockListProto cleanupList) {
//skip for testing
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java
index e8c2b9c..f57fa85 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java
@@ -52,11 +52,6 @@ public abstract class MockWorkerContext implements TajoWorker.WorkerContext {
}
@Override
- public TaskRunnerManager getTaskRunnerManager() {
- return null;
- }
-
- @Override
public CatalogService getCatalog() {
return null;
}
@@ -77,11 +72,6 @@ public abstract class MockWorkerContext implements TajoWorker.WorkerContext {
}
@Override
- public QueryCoordinatorProtocol.ClusterResourceSummary getClusterResource() {
- return null;
- }
-
- @Override
public TajoSystemMetrics getWorkerSystemMetrics() {
if (tajoSystemMetrics == null) {
@@ -115,15 +105,5 @@ public abstract class MockWorkerContext implements TajoWorker.WorkerContext {
public void cleanupTemporalDirectories() {
}
-
- @Override
- public void setClusterResource(QueryCoordinatorProtocol.ClusterResourceSummary clusterResource) {
-
- }
-
- @Override
- public void setNumClusterNodes(int numClusterNodes) {
-
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
index 65627c1..a91fc30 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestFetcher.java
@@ -104,13 +104,13 @@ public class TestFetcher {
@Test
public void testAdjustFetchProcess() {
- assertEquals(0.0f, LegacyTaskImpl.adjustFetchProcess(0, 0), 0);
- assertEquals(0.0f, LegacyTaskImpl.adjustFetchProcess(10, 10), 0);
- assertEquals(0.05f, LegacyTaskImpl.adjustFetchProcess(10, 9), 0);
- assertEquals(0.1f, LegacyTaskImpl.adjustFetchProcess(10, 8), 0);
- assertEquals(0.25f, LegacyTaskImpl.adjustFetchProcess(10, 5), 0);
- assertEquals(0.45f, LegacyTaskImpl.adjustFetchProcess(10, 1), 0);
- assertEquals(0.5f, LegacyTaskImpl.adjustFetchProcess(10, 0), 0);
+ assertEquals(0.0f, TaskImpl.adjustFetchProcess(0, 0), 0);
+ assertEquals(0.0f, TaskImpl.adjustFetchProcess(10, 10), 0);
+ assertEquals(0.05f, TaskImpl.adjustFetchProcess(10, 9), 0);
+ assertEquals(0.1f, TaskImpl.adjustFetchProcess(10, 8), 0);
+ assertEquals(0.25f, TaskImpl.adjustFetchProcess(10, 5), 0);
+ assertEquals(0.45f, TaskImpl.adjustFetchProcess(10, 1), 0);
+ assertEquals(0.5f, TaskImpl.adjustFetchProcess(10, 0), 0);
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
deleted file mode 100644
index df6d714..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.service.Service;
-import org.apache.tajo.TajoProtos;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.client.TajoClient;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.master.QueryInfo;
-import org.apache.tajo.master.TajoMaster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-public class TestHistory {
- private static TajoTestingCluster cluster;
- private static TajoMaster master;
- private static TajoConf conf;
- private static TajoClient client;
-
- @BeforeClass
- public static void setUp() throws Exception {
- cluster = new TajoTestingCluster();
- cluster.startMiniClusterInLocal(1);
- master = cluster.getMaster();
- conf = cluster.getConfiguration();
- client = cluster.newTajoClient();
- File file = TPCH.getDataFile("lineitem");
- client.executeQueryAndGetResult("create external table default.lineitem (l_orderkey int, l_partkey int) "
- + "using text location 'file://" + file.getAbsolutePath() + "'");
- assertTrue(client.existTable("default.lineitem"));
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- if (client != null) client.close();
- if (cluster != null) cluster.shutdownMiniCluster();
- }
-
- @Test
- public final void testTaskRunnerHistory() throws IOException, ServiceException, InterruptedException {
- int beforeFinishedQueriesCount = master.getContext().getQueryJobManager().getFinishedQueries().size();
- client.executeQueryAndGetResult("select count(*) from lineitem");
-
- Collection<QueryInfo> finishedQueries = master.getContext().getQueryJobManager().getFinishedQueries();
- assertTrue(finishedQueries.size() > beforeFinishedQueriesCount);
-
- TajoWorker worker = cluster.getTajoWorkers().get(0);
- TaskRunnerManager taskRunnerManager = worker.getWorkerContext().getTaskRunnerManager();
- assertNotNull(taskRunnerManager);
-
-
- Collection<TaskRunnerHistory> histories = taskRunnerManager.getExecutionBlockHistories();
- assertTrue(histories.size() > 0);
-
- TaskRunnerHistory history = histories.iterator().next();
- assertEquals(Service.STATE.STOPPED, history.getState());
- TaskRunnerHistory fromProto = new TaskRunnerHistory(history.getProto());
- assertEquals(history.getExecutionBlockId(), fromProto.getExecutionBlockId());
- assertEquals(history.getFinishTime(), fromProto.getFinishTime());
- assertEquals(history.getStartTime(), fromProto.getStartTime());
- assertEquals(history.getState(), fromProto.getState());
- assertEquals(history.getContainerId(), fromProto.getContainerId());
- assertEquals(history.getProto().getTaskHistoriesCount(), fromProto.getProto().getTaskHistoriesCount());
- }
-
- @Test
- public final void testTaskHistory() throws IOException, ServiceException, InterruptedException {
- int beforeFinishedQueriesCount = master.getContext().getQueryJobManager().getFinishedQueries().size();
- client.executeQueryAndGetResult("select count(*) from lineitem");
-
- Collection<QueryInfo> finishedQueries = master.getContext().getQueryJobManager().getFinishedQueries();
- assertTrue(finishedQueries.size() > beforeFinishedQueriesCount);
-
- TajoWorker worker = cluster.getTajoWorkers().get(0);
- TaskRunnerManager taskRunnerManager = worker.getWorkerContext().getTaskRunnerManager();
- assertNotNull(taskRunnerManager);
-
-
- Collection<TaskRunnerHistory> histories = taskRunnerManager.getExecutionBlockHistories();
- assertTrue(histories.size() > 0);
-
- TaskRunnerHistory history = histories.iterator().next();
-
- assertTrue(history.size() > 0);
- assertEquals(Service.STATE.STOPPED, history.getState());
-
- Map.Entry<TaskAttemptId, TaskHistory> entry =
- history.getTaskHistoryMap().entrySet().iterator().next();
-
- TaskAttemptId taskAttemptId = entry.getKey();
- TaskHistory taskHistory = entry.getValue();
-
- assertEquals(TajoProtos.TaskAttemptState.TA_SUCCEEDED, taskHistory.getState());
- assertEquals(taskAttemptId, taskHistory.getTaskAttemptId());
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java
index 0f58854..7539beb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java
@@ -22,26 +22,26 @@ import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.tajo.*;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.worker.event.NodeResourceAllocateEvent;
import org.apache.tajo.worker.event.NodeResourceDeallocateEvent;
-import org.junit.*;
+import org.apache.tajo.worker.event.NodeResourceEvent;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.tajo.ResourceProtos.*;
import static org.junit.Assert.*;
-
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
public class TestNodeResourceManager {
private MockNodeResourceManager resourceManager;
@@ -65,8 +65,9 @@ public class TestNodeResourceManager {
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES, 4);
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB,
taskMemory * conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES));
- conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS_NUM, 4);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS, 4);
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM, 1);
+ conf.setIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM, 2);
dispatcher = new AsyncDispatcher();
taskDispatcher = new AsyncDispatcher();
@@ -79,6 +80,21 @@ public class TestNodeResourceManager {
}
@Override
+ public TaskManager getTaskManager() {
+ return taskManager;
+ }
+
+ @Override
+ public TaskExecutor getTaskExecuor() {
+ return taskExecutor;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return resourceManager;
+ }
+
+ @Override
public WorkerConnectionInfo getConnectionInfo() {
if (workerConnectionInfo == null) {
workerConnectionInfo = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
@@ -87,10 +103,10 @@ public class TestNodeResourceManager {
}
};
- taskManager = new MockTaskManager(new Semaphore(0), taskDispatcher, workerContext, dispatcher.getEventHandler());
- taskExecutor = new MockTaskExecutor(new Semaphore(0), taskManager, dispatcher.getEventHandler());
- resourceManager = new MockNodeResourceManager(new Semaphore(0), dispatcher, taskDispatcher.getEventHandler());
- statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext, resourceManager);
+ taskManager = new MockTaskManager(new Semaphore(0), taskDispatcher, workerContext);
+ taskExecutor = new MockTaskExecutor(new Semaphore(0), workerContext);
+ resourceManager = new MockNodeResourceManager(new Semaphore(0), dispatcher, workerContext);
+ statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext);
service = new CompositeService("MockService") {
@Override
@@ -103,6 +119,12 @@ public class TestNodeResourceManager {
addIfService(statusUpdater);
super.serviceInit(conf);
}
+
+ @Override
+ protected void serviceStop() throws Exception {
+ workerContext.getWorkerSystemMetrics().stop();
+ super.serviceStop();
+ }
};
service.init(conf);
@@ -119,8 +141,8 @@ public class TestNodeResourceManager {
int requestSize = 4;
resourceManager.setTaskHandlerEvent(false); //skip task execution
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
requestProto.setExecutionBlockId(ebId.getProto());
@@ -129,7 +151,7 @@ public class TestNodeResourceManager {
dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
- BatchAllocationResponseProto responseProto = callFuture.get();
+ BatchAllocationResponse responseProto = callFuture.get();
assertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
// allocated all
assertEquals(0, responseProto.getCancellationTaskCount());
@@ -142,8 +164,8 @@ public class TestNodeResourceManager {
int overSize = 10;
resourceManager.setTaskHandlerEvent(false); //skip task execution
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
requestProto.setExecutionBlockId(ebId.getProto());
@@ -152,7 +174,7 @@ public class TestNodeResourceManager {
MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize + overSize));
dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
- BatchAllocationResponseProto responseProto = callFuture.get();
+ BatchAllocationResponse responseProto = callFuture.get();
assertEquals(overSize, responseProto.getCancellationTaskCount());
}
@@ -162,8 +184,8 @@ public class TestNodeResourceManager {
int requestSize = 4;
resourceManager.setTaskHandlerEvent(false); //skip task execution
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
requestProto.setExecutionBlockId(ebId.getProto());
@@ -172,14 +194,15 @@ public class TestNodeResourceManager {
dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
- BatchAllocationResponseProto responseProto = callFuture.get();
+ BatchAllocationResponse responseProto = callFuture.get();
assertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
assertEquals(0, responseProto.getCancellationTaskCount());
//deallocate
- for(TaskAllocationRequestProto allocationRequestProto : requestProto.getTaskRequestList()) {
+ for(TaskAllocationProto allocationRequestProto : requestProto.getTaskRequestList()) {
// direct invoke handler for testing
- resourceManager.handle(new NodeResourceDeallocateEvent(allocationRequestProto.getResource()));
+ resourceManager.handle(new NodeResourceDeallocateEvent(
+ allocationRequestProto.getResource(), NodeResourceEvent.ResourceType.TASK));
}
assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
@@ -195,33 +218,22 @@ public class TestNodeResourceManager {
final AtomicInteger totalCanceled = new AtomicInteger();
final ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
- final Queue<TaskAllocationRequestProto>
+ final Queue<TaskAllocationProto>
totalTasks = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, taskSize);
- // first request with starting ExecutionBlock
- TajoWorkerProtocol.RunExecutionBlockRequestProto.Builder
- ebRequestProto = TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder();
- ebRequestProto.setExecutionBlockId(ebId.getProto())
- .setQueryMaster(workerContext.getConnectionInfo().getProto())
- .setNodeId(workerContext.getConnectionInfo().getHost() + ":" +
- workerContext.getConnectionInfo().getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
- .setPlanJson("test")
- .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
-
- TaskAllocationRequestProto task = totalTasks.poll();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+
+ TaskAllocationProto task = totalTasks.poll();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.addTaskRequest(task);
requestProto.setExecutionBlockId(ebId.getProto());
- requestProto.setExecutionBlockRequest(ebRequestProto.build());
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
assertTrue(callFuture.get().getCancellationTaskCount() == 0);
totalComplete.incrementAndGet();
// start parallel request
ExecutorService executor = Executors.newFixedThreadPool(parallelCount);
+
List<Future> futureList = Lists.newArrayList();
long startTime = System.currentTimeMillis();
@@ -231,18 +243,18 @@ public class TestNodeResourceManager {
public void run() {
int complete = 0;
while (true) {
- TaskAllocationRequestProto task = totalTasks.poll();
+ TaskAllocationProto task = totalTasks.poll();
if (task == null) break;
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.addTaskRequest(task);
requestProto.setExecutionBlockId(ebId.getProto());
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
try {
- BatchAllocationResponseProto proto = callFuture.get();
+ BatchAllocationResponse proto = callFuture.get();
if (proto.getCancellationTaskCount() > 0) {
totalTasks.addAll(proto.getCancellationTaskList());
totalCanceled.addAndGet(proto.getCancellationTaskCount());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java
index af40554..707aa49 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java
@@ -18,11 +18,11 @@
package org.apache.tajo.worker;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.TajoResourceTrackerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.rm.Worker;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.worker.event.NodeStatusEvent;
import org.junit.After;
@@ -30,13 +30,19 @@ import org.junit.Before;
import org.junit.Test;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+
+import static org.apache.tajo.ResourceProtos.NodeHeartbeatRequest;
import static org.junit.Assert.*;
public class TestNodeStatusUpdater {
private NodeResourceManager resourceManager;
private MockNodeStatusUpdater statusUpdater;
+ private MockTaskManager taskManager;
private AsyncDispatcher dispatcher;
+ private AsyncDispatcher taskDispatcher;
+ private CompositeService service;
private TajoConf conf;
private TajoWorker.WorkerContext workerContext;
@@ -45,6 +51,9 @@ public class TestNodeStatusUpdater {
public void setup() {
conf = new TajoConf();
conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES, 2);
+ conf.setIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM, 2);
+
workerContext = new MockWorkerContext() {
WorkerConnectionInfo workerConnectionInfo;
@@ -54,6 +63,21 @@ public class TestNodeStatusUpdater {
}
@Override
+ public TaskManager getTaskManager() {
+ return taskManager;
+ }
+
+ @Override
+ public TaskExecutor getTaskExecuor() {
+ return null;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return resourceManager;
+ }
+
+ @Override
public WorkerConnectionInfo getConnectionInfo() {
if (workerConnectionInfo == null) {
workerConnectionInfo = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
@@ -62,27 +86,48 @@ public class TestNodeStatusUpdater {
}
};
- conf.setIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_INTERVAL, 1000);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL, 1000);
dispatcher = new AsyncDispatcher();
- dispatcher.init(conf);
- dispatcher.start();
+ resourceManager = new NodeResourceManager(dispatcher, workerContext);
+ taskDispatcher = new AsyncDispatcher();
+ taskManager = new MockTaskManager(new Semaphore(0), taskDispatcher, workerContext) {
+ @Override
+ public int getRunningTasks() {
+ return 0;
+ }
+ };
+
+ service = new CompositeService("MockService") {
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ addIfService(dispatcher);
+ addIfService(taskDispatcher);
+ addIfService(taskManager);
+ addIfService(resourceManager);
+ addIfService(statusUpdater);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ workerContext.getWorkerSystemMetrics().stop();
+ super.serviceStop();
+ }
+ };
- resourceManager = new NodeResourceManager(dispatcher, null);
- resourceManager.init(conf);
- resourceManager.start();
+ service.init(conf);
+ service.start();
}
@After
public void tearDown() {
- resourceManager.stop();
- if (statusUpdater != null) statusUpdater.stop();
- dispatcher.stop();
+ service.stop();
}
@Test(timeout = 20000)
public void testNodeMembership() throws Exception {
CountDownLatch barrier = new CountDownLatch(1);
- statusUpdater = new MockNodeStatusUpdater(barrier, workerContext, resourceManager);
+ statusUpdater = new MockNodeStatusUpdater(barrier, workerContext);
statusUpdater.init(conf);
statusUpdater.start();
@@ -100,16 +145,18 @@ public class TestNodeStatusUpdater {
@Test(timeout = 20000)
public void testPing() throws Exception {
CountDownLatch barrier = new CountDownLatch(2);
- statusUpdater = new MockNodeStatusUpdater(barrier, workerContext, resourceManager);
+ statusUpdater = new MockNodeStatusUpdater(barrier, workerContext);
statusUpdater.init(conf);
statusUpdater.start();
MockNodeStatusUpdater.MockResourceTracker resourceTracker = statusUpdater.getResourceTracker();
barrier.await();
- TajoResourceTrackerProtocol.NodeHeartbeatRequestProto lastRequest = resourceTracker.getLastRequest();
+ NodeHeartbeatRequest lastRequest = resourceTracker.getLastRequest();
assertTrue(lastRequest.hasWorkerId());
- assertFalse(lastRequest.hasAvailableResource());
+ assertTrue(lastRequest.hasAvailableResource());
+ assertTrue(lastRequest.hasRunningTasks());
+ assertTrue(lastRequest.hasRunningQueryMasters());
assertFalse(lastRequest.hasTotalResource());
assertFalse(lastRequest.hasConnectionInfo());
}
@@ -117,12 +164,12 @@ public class TestNodeStatusUpdater {
@Test(timeout = 20000)
public void testResourceReport() throws Exception {
CountDownLatch barrier = new CountDownLatch(2);
- statusUpdater = new MockNodeStatusUpdater(barrier, workerContext, resourceManager);
+ statusUpdater = new MockNodeStatusUpdater(barrier, workerContext);
statusUpdater.init(conf);
statusUpdater.start();
assertEquals(0, statusUpdater.getQueueSize());
- for (int i = 0; i < statusUpdater.getQueueingLimit(); i++) {
+ for (int i = 0; i < statusUpdater.getQueueingThreshold(); i++) {
dispatcher.getEventHandler().handle(new NodeStatusEvent(NodeStatusEvent.EventType.REPORT_RESOURCE));
}
barrier.await();
@@ -132,7 +179,7 @@ public class TestNodeStatusUpdater {
@Test(timeout = 20000)
public void testFlushResourceReport() throws Exception {
CountDownLatch barrier = new CountDownLatch(2);
- statusUpdater = new MockNodeStatusUpdater(barrier, workerContext, resourceManager);
+ statusUpdater = new MockNodeStatusUpdater(barrier, workerContext);
statusUpdater.init(conf);
statusUpdater.start();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
index 98b187b..9b6af68 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java
@@ -19,55 +19,36 @@
package org.apache.tajo.worker;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.*;
-import org.apache.tajo.annotation.ThreadSafe;
-import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.plan.serder.PlanProto;
-import org.apache.tajo.querymaster.QueryMaster;
-import org.apache.tajo.querymaster.QueryMasterManagerService;
-import org.apache.tajo.resource.NodeResources;
import org.apache.tajo.rpc.CallFuture;
-import org.apache.tajo.service.ServiceTracker;
-import org.apache.tajo.storage.HashShuffleAppenderManager;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.history.HistoryReader;
-import org.apache.tajo.util.history.HistoryWriter;
-import org.apache.tajo.util.metrics.TajoSystemMetrics;
-import org.apache.tajo.worker.event.ExecutionBlockStartEvent;
-import org.apache.tajo.worker.event.ExecutionBlockStopEvent;
import org.apache.tajo.worker.event.NodeResourceAllocateEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
-import java.util.Queue;
+import java.util.List;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
-import static org.junit.Assert.*;
+import static org.apache.tajo.ResourceProtos.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestTaskExecutor {
private NodeResourceManager resourceManager;
private NodeStatusUpdater statusUpdater;
private TaskManager taskManager;
- private TaskExecutor taskExecutor;
+ private MyTaskExecutor taskExecutor;
private AsyncDispatcher dispatcher;
private AsyncDispatcher taskDispatcher;
private TajoWorker.WorkerContext workerContext;
@@ -81,6 +62,8 @@ public class TestTaskExecutor {
public void setup() {
conf = new TajoConf();
conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES, 2);
+ conf.setIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM, 2);
dispatcher = new AsyncDispatcher();
taskDispatcher = new AsyncDispatcher();
@@ -93,6 +76,21 @@ public class TestTaskExecutor {
}
@Override
+ public TaskManager getTaskManager() {
+ return taskManager;
+ }
+
+ @Override
+ public org.apache.tajo.worker.TaskExecutor getTaskExecuor() {
+ return taskExecutor;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return resourceManager;
+ }
+
+ @Override
public WorkerConnectionInfo getConnectionInfo() {
if (workerConnectionInfo == null) {
workerConnectionInfo = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
@@ -103,10 +101,10 @@ public class TestTaskExecutor {
barrier = new Semaphore(0);
resourceManagerBarrier = new Semaphore(0);
- taskManager = new MockTaskManager(new Semaphore(0), taskDispatcher, workerContext, dispatcher.getEventHandler());
- taskExecutor = new TaskExecutor(barrier, taskManager, dispatcher.getEventHandler());
- resourceManager = new MockNodeResourceManager(resourceManagerBarrier, dispatcher, taskDispatcher.getEventHandler());
- statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext, resourceManager);
+ taskManager = new MockTaskManager(new Semaphore(0), taskDispatcher, workerContext);
+ taskExecutor = new MyTaskExecutor(barrier, workerContext);
+ resourceManager = new MockNodeResourceManager(resourceManagerBarrier, dispatcher, workerContext);
+ statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext);
service = new CompositeService("MockService") {
@Override
@@ -141,24 +139,12 @@ public class TestTaskExecutor {
public void testTaskRequest() throws Exception {
int requestSize = 1;
- RunExecutionBlockRequestProto.Builder
- ebRequestProto = RunExecutionBlockRequestProto.newBuilder();
QueryId qid = LocalTajoTestingUtility.newQueryId();
ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1);
- ebRequestProto.setExecutionBlockId(ebId.getProto())
- .setQueryMaster(workerContext.getConnectionInfo().getProto())
- .setNodeId(workerContext.getConnectionInfo().getHost() + ":"
- + workerContext.getConnectionInfo().getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
- .setPlanJson("test")
- .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
-
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.setExecutionBlockId(ebId.getProto());
- requestProto.setExecutionBlockRequest(ebRequestProto.build());
assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, 10, requestSize));
@@ -181,24 +167,12 @@ public class TestTaskExecutor {
public void testTaskException() throws Exception {
int requestSize = 1;
- RunExecutionBlockRequestProto.Builder
- ebRequestProto = RunExecutionBlockRequestProto.newBuilder();
QueryId qid = LocalTajoTestingUtility.newQueryId();
ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1);
- ebRequestProto.setExecutionBlockId(ebId.getProto())
- .setQueryMaster(workerContext.getConnectionInfo().getProto())
- .setNodeId(workerContext.getConnectionInfo().getHost()+":"
- + workerContext.getConnectionInfo().getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
- .setPlanJson("test")
- .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
-
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.setExecutionBlockId(ebId.getProto());
- requestProto.setExecutionBlockRequest(ebRequestProto.build());
assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, 10, requestSize));
@@ -218,12 +192,12 @@ public class TestTaskExecutor {
assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
}
- class TaskExecutor extends MockTaskExecutor {
+ class MyTaskExecutor extends MockTaskExecutor {
int completeTasks;
AtomicBoolean throwException = new AtomicBoolean();
- public TaskExecutor(Semaphore barrier, TaskManager taskManager, EventHandler rmEventHandler) {
- super(barrier, taskManager, rmEventHandler);
+ public MyTaskExecutor(Semaphore barrier, TajoWorker.WorkerContext workerContext) {
+ super(barrier, workerContext);
}
@Override
@@ -233,7 +207,7 @@ public class TestTaskExecutor {
}
@Override
- protected Task createTask(final ExecutionBlockContext context, TajoWorkerProtocol.TaskRequestProto taskRequest) {
+ protected Task createTask(final ExecutionBlockContext context, TaskRequestProto taskRequest) {
final TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
final TaskAttemptContext taskAttemptContext = new TaskAttemptContext(null, context, taskAttemptId, null, null);
@@ -314,8 +288,8 @@ public class TestTaskExecutor {
}
@Override
- public TajoWorkerProtocol.TaskStatusProto getReport() {
- TajoWorkerProtocol.TaskStatusProto.Builder builder = TajoWorkerProtocol.TaskStatusProto.newBuilder();
+ public TaskStatusProto getReport() {
+ TaskStatusProto.Builder builder = TaskStatusProto.newBuilder();
builder.setWorkerName("localhost:0");
builder.setId(taskAttemptContext.getTaskId().getProto())
.setProgress(taskAttemptContext.getProgress())
@@ -324,6 +298,16 @@ public class TestTaskExecutor {
builder.setInputStats(new TableStats().getProto());
return builder.build();
}
+
+ @Override
+ public TaskHistory createTaskHistory() {
+ return null;
+ }
+
+ @Override
+ public List<Fetcher> getFetchers() {
+ return null;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java
index 8bca489..630f3f7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java
@@ -21,24 +21,27 @@ package org.apache.tajo.worker;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.tajo.*;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.LocalTajoTestingUtility;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.plan.serder.PlanProto;
+import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.worker.event.ExecutionBlockStartEvent;
import org.apache.tajo.worker.event.ExecutionBlockStopEvent;
import org.apache.tajo.worker.event.NodeResourceAllocateEvent;
+import org.apache.tajo.worker.event.TaskStartEvent;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.util.concurrent.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
+import static org.apache.tajo.ResourceProtos.*;
import static org.junit.Assert.*;
public class TestTaskManager {
@@ -65,8 +68,9 @@ public class TestTaskManager {
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES, 4);
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB,
taskMemory * conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES));
- conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS_NUM, 4);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS, 4);
conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM, 1);
+ conf.setIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM, 2);
dispatcher = new AsyncDispatcher();
taskDispatcher = new AsyncDispatcher();
@@ -80,6 +84,21 @@ public class TestTaskManager {
}
@Override
+ public TaskManager getTaskManager() {
+ return taskManager;
+ }
+
+ @Override
+ public TaskExecutor getTaskExecuor() {
+ return taskExecutor;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return resourceManager;
+ }
+
+ @Override
public WorkerConnectionInfo getConnectionInfo() {
if (workerConnectionInfo == null) {
workerConnectionInfo = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
@@ -88,10 +107,10 @@ public class TestTaskManager {
}
};
barrier = new Semaphore(0);
- taskManager = new MockTaskManager(barrier, taskDispatcher, workerContext, dispatcher.getEventHandler());
- taskExecutor = new MockTaskExecutor(new Semaphore(0), taskManager, dispatcher.getEventHandler());
- resourceManager = new NodeResourceManager(dispatcher, taskDispatcher.getEventHandler());
- statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext, resourceManager);
+ taskManager = new MockTaskManager(barrier, taskDispatcher, workerContext);
+ taskExecutor = new MockTaskExecutor(new Semaphore(0), workerContext);
+ resourceManager = new NodeResourceManager(dispatcher, workerContext);
+ statusUpdater = new MockNodeStatusUpdater(new CountDownLatch(0), workerContext);
service = new CompositeService("MockService") {
@Override
@@ -125,25 +144,12 @@ public class TestTaskManager {
@Test(timeout = 10000)
public void testExecutionBlockStart() throws Exception {
int requestSize = 1;
-
- TajoWorkerProtocol.RunExecutionBlockRequestProto.Builder
- ebRequestProto = TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder();
QueryId qid = LocalTajoTestingUtility.newQueryId();
ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1);
- ebRequestProto.setExecutionBlockId(ebId.getProto())
- .setQueryMaster(workerContext.getConnectionInfo().getProto())
- .setNodeId(workerContext.getConnectionInfo().getHost() + ":"
- + workerContext.getConnectionInfo().getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
- .setPlanJson("test")
- .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
-
- CallFuture<BatchAllocationResponseProto> callFuture = new CallFuture<BatchAllocationResponseProto>();
- BatchAllocationRequestProto.Builder requestProto = BatchAllocationRequestProto.newBuilder();
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.setExecutionBlockId(ebId.getProto());
- requestProto.setExecutionBlockRequest(ebRequestProto.build());
assertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize));
@@ -158,21 +164,14 @@ public class TestTaskManager {
@Test(timeout = 10000)
public void testExecutionBlockStop() throws Exception {
- TajoWorkerProtocol.RunExecutionBlockRequestProto.Builder
- ebRequestProto = TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder();
QueryId qid = LocalTajoTestingUtility.newQueryId();
ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1);
+ TaskAllocationProto requestProto =
+ MockNodeResourceManager.createTaskRequests(ebId, taskMemory, 1).poll();
- ebRequestProto.setExecutionBlockId(ebId.getProto())
- .setQueryMaster(workerContext.getConnectionInfo().getProto())
- .setNodeId(workerContext.getConnectionInfo().getHost()+":"
- + workerContext.getConnectionInfo().getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
- .setPlanJson("test")
- .setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
+ taskDispatcher.getEventHandler().handle(new TaskStartEvent(requestProto.getTaskRequest(),
+ new NodeResource(requestProto.getResource())));
- taskDispatcher.getEventHandler().handle(new ExecutionBlockStartEvent(ebRequestProto.build()));
assertTrue(barrier.tryAcquire(3, TimeUnit.SECONDS));
assertNotNull(taskManager.getExecutionBlockContext(ebId));
assertEquals(ebId, taskManager.getExecutionBlockContext(ebId).getExecutionBlockId());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-docs/src/main/sphinx/configuration/worker_configuration.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/configuration/worker_configuration.rst b/tajo-docs/src/main/sphinx/configuration/worker_configuration.rst
index 2713948..0756c0a 100644
--- a/tajo-docs/src/main/sphinx/configuration/worker_configuration.rst
+++ b/tajo-docs/src/main/sphinx/configuration/worker_configuration.rst
@@ -43,21 +43,26 @@ Worker Resources
==========================================================
Each worker can execute multiple tasks simultaneously.
-In Tajo, users can specify the total size of memory and the number of disks for each worker. Available resources affect how many tasks are executed simultaneously.
+
+In Tajo, users can specify the number of cpu cores, the total size of memory and the number of disks for each worker. Available resources affect how many tasks are executed simultaneously.
+CPU cores are a unit for expressing CPU parallelism, the unit for memory is megabytes and the unit for disks is the number of disk
In order to specify the resource capacity of each worker, you should add the following configs to ``tajo-site.xml`` :
-================================= ========================== =================== =========================
- property name description value type default value
-================================= ========================== =================== =========================
- tajo.worker.resource.cpu-cores the number of cpu cores integer 1
- tajo.worker.resource.memory-mb memory size (MB) integer 1024
- tajo.worker.resource.disks the number of disks integer 1
-================================= ========================== =================== =========================
+=================================== ============= ====================== =================================
+ property name value type default value description
+=================================== ============= ====================== =================================
+ tajo.worker.resource.cpu-cores Integer available cpu-cores the number of cpu cores
+ tajo.worker.resource.memory-mb Integer available jvm heap memory size (MB)
+ tajo.worker.resource.disks Integer 1 the number of disks
+ tajo.task.resource.min.memory-mb Integer 500 minimum allocatable memory per task
+ tajo.qm.resource.min.memory-mb Integer 500 minimum allocatable memory per query
+=================================== ============= ====================== =================================
.. note::
- Currently, QueryMaster requests 512MB memory and 0.5 disk per task for the backward compatibility.
+ Currently, QueryMaster requests 500MB memory and 1 cpu-core per task for the backward compatibility.
+ If you want to give more memory, you can set to ``tajo.qm.resource.min.memory-mb``
.. note::
@@ -68,7 +73,14 @@ In order to specify the resource capacity of each worker, you should add the fol
Example
------------
-Assume that you want to give 5120 MB memory, 4 disks, and 24 cores on each worker. The example configuration is as follows:
+Assume that you want to give 15GB Jvm heap, 1GB memory per task, 4 disks, and 12 cores on each worker. The example configuration is as follows:
+
+``tajo-env.sh``
+
+.. code-block:: bash
+
+ export TAJO_WORKER_HEAPSIZE=15000
+
``tajo-site.xml``
@@ -76,35 +88,63 @@ Assume that you want to give 5120 MB memory, 4 disks, and 24 cores on each worke
<property>
<name>tajo.worker.resource.tajo.worker.resource.cpu-cores</name>
- <value>24</value>
+ <value>12</value>
</property>
<property>
- <name>tajo.worker.resource.memory-mb</name>
- <value>5120</value>
+ <name>tajo.task.resource.min.memory-mb</name>
+ <value>1000</value>
</property>
<property>
- <name>tajo.worker.resource.tajo.worker.resource.disks</name>
- <value>4.0</value>
- </property>
+ <name>tajo.worker.resource.disks</name>
+ <value>4</value>
+ </property>
+
---------------------
- Dedicated Mode
---------------------
-Tajo provides a dedicated mode that allows each worker in a Tajo cluster to use whole available system resources including cpu-cores, memory, and disks. For this mode, a user should add the following config to ``tajo-site.xml`` :
+* Example with HDFS
+
+``tajo-env.sh``
+
+.. code-block:: bash
+
+ export TAJO_WORKER_HEAPSIZE=15000
+
+
+``tajo-site.xml``
.. code-block:: xml
+ <property>
+ <name>tajo.task.resource.min.memory-mb</name>
+ <value>1000</value>
+ </property>
+
<property>
- <name>tajo.worker.resource.dedicated</name>
+ <name>tajo.worker.resource.dfs-dir-aware</name>
<value>true</value>
</property>
-In addition, it can limit the memory capacity used for Tajo worker as follows:
-=============================================== ================================================ =================== =======================
- property name description value type default value
-=============================================== ================================================ =================== =======================
- tajo.worker.resource.dedicated-memory-ratio how much memory to be used in whole memory float 0.8
-=============================================== ================================================ =================== =======================
\ No newline at end of file
+* Example with S3
+
+``tajo-env.sh``
+
+.. code-block:: bash
+
+ export TAJO_WORKER_HEAPSIZE=15000
+
+
+``tajo-site.xml``
+
+.. code-block:: xml
+
+ <property>
+ <name>tajo.task.resource.min.memory-mb</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>tajo.worker.resource.disk.parallel-execution.num</name>
+ <value>4</value>
+ </property>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index e314f99..1aee05f 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -21,19 +21,19 @@ package org.apache.tajo.plan.util;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.tajo.OverridableConf;
-import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.DataType;
-import org.apache.tajo.plan.*;
+import org.apache.tajo.plan.InvalidQueryException;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.PlanningException;
+import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor;
import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
-import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index 7855acd..537d180 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -24,6 +24,7 @@ option java_generate_equals_and_hash = true;
import "PrimitiveProtos.proto";
import "CatalogProtos.proto";
import "DataTypes.proto";
+import "TajoIdProtos.proto";
enum NodeType {
SET_SESSION = 0;
@@ -505,4 +506,128 @@ enum TransmitType {
PUSH_TRANSMIT = 0;
PULL_TRANSMIT = 1;
FILE_WRITE = 2;
+}
+
+message DataChannelProto {
+ required ExecutionBlockIdProto srcId = 1;
+ required ExecutionBlockIdProto targetId = 2;
+
+ required TransmitType transmitType = 3 [default = PULL_TRANSMIT];
+ required ShuffleType shuffleType = 4;
+
+ optional SchemaProto schema = 5;
+
+ repeated ColumnProto shuffleKeys = 7;
+ optional int32 numOutputs = 9 [default = 1];
+
+ optional string storeType = 10;
+}
+
+message EnforcerProto {
+ repeated EnforceProperty properties = 1;
+}
+
+message EnforceProperty {
+ enum EnforceType {
+ SORTED_INPUT = 0;
+ OUTPUT_DISTINCT = 1;
+ GROUP_BY = 2;
+ JOIN = 3;
+ SORT = 4;
+ BROADCAST = 5;
+ COLUMN_PARTITION = 6;
+ DISTINCT_GROUP_BY = 7;
+ }
+
+ // Identifies which field is filled in.
+ required EnforceType type = 1;
+
+ // One of the following will be filled in.
+ optional SortedInputEnforce sortedInput = 2;
+ optional OutputDistinctEnforce outputDistinct = 3;
+ optional GroupbyEnforce groupby = 4;
+ optional JoinEnforce join = 5;
+ optional SortEnforce sort = 6;
+ optional BroadcastEnforce broadcast = 7;
+ optional ColumnPartitionEnforcer columnPartition = 8;
+ optional DistinctGroupbyEnforcer distinct = 9;
+}
+
+message SortedInputEnforce {
+ required string tableName = 1;
+ repeated SortSpecProto sortSpecs = 2;
+}
+
+message OutputDistinctEnforce {
+}
+
+message JoinEnforce {
+ enum JoinAlgorithm {
+ NESTED_LOOP_JOIN = 0;
+ BLOCK_NESTED_LOOP_JOIN = 1;
+ IN_MEMORY_HASH_JOIN = 2;
+ HYBRID_HASH_JOIN = 3;
+ MERGE_JOIN = 4;
+ }
+
+ required int32 nodeId = 1;
+ required JoinAlgorithm algorithm = 2;
+}
+
+message GroupbyEnforce {
+ enum GroupbyAlgorithm {
+ HASH_AGGREGATION = 0;
+ SORT_AGGREGATION = 1;
+ }
+
+ required int32 nodeId = 1;
+ required GroupbyAlgorithm algorithm = 2;
+ repeated SortSpecProto sortSpecs = 3;
+}
+
+message SortEnforce {
+ enum SortAlgorithm {
+ IN_MEMORY_SORT = 0;
+ MERGE_SORT = 1;
+ }
+
+ required int32 nodeId = 1;
+ required SortAlgorithm algorithm = 2;
+}
+
+message BroadcastEnforce {
+ required string tableName = 1;
+}
+
+message ColumnPartitionEnforcer {
+ enum ColumnPartitionAlgorithm {
+ HASH_PARTITION = 0;
+ SORT_PARTITION = 1;
+ }
+
+ required int32 nodeId = 1;
+ required ColumnPartitionAlgorithm algorithm = 2;
+}
+
+message DistinctGroupbyEnforcer {
+ enum DistinctAggregationAlgorithm {
+ HASH_AGGREGATION = 0;
+ SORT_AGGREGATION = 1;
+ }
+
+ enum MultipleAggregationStage {
+ FIRST_STAGE = 0;
+ SECOND_STAGE = 1;
+ THRID_STAGE = 3;
+ }
+
+ message SortSpecArray {
+ required int32 nodeId = 1;
+ repeated SortSpecProto sortSpecs = 2;
+ }
+ required int32 nodeId = 1;
+ required DistinctAggregationAlgorithm algorithm = 2;
+ repeated SortSpecArray sortSpecArrays = 3;
+ required bool isMultipleAggregation = 4 [default = false];
+ optional MultipleAggregationStage multipleAggregationStage = 5;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index a98cc13..d7e2f91 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -38,7 +38,7 @@
<tajo.version>0.11.0-SNAPSHOT</tajo.version>
<hbase.version>0.98.7-hadoop2</hbase.version>
<hive.version>1.1.0</hive.version>
- <netty.version>4.0.25.Final</netty.version>
+ <netty.version>4.0.29.Final</netty.version>
<jersey.version>2.6</jersey.version>
<tajo.root>${project.parent.relativePath}/..</tajo.root>
<extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
index 134b3cf..88ffaf6 100644
--- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
+++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcServer.java
@@ -131,10 +131,8 @@ public class AsyncRpcServer extends NettyServerBase {
RemoteCallException callException = (RemoteCallException) cause;
ctx.writeAndFlush(callException.getResponse());
- if(LOG.isDebugEnabled()) {
- Throwable rootCause = ExceptionUtils.getRootCause(cause);
- LOG.error(ExceptionUtils.getMessage(rootCause), rootCause);
- }
+ Throwable rootCause = ExceptionUtils.getRootCause(cause);
+ LOG.error(ExceptionUtils.getMessage(rootCause), rootCause);
} else {
/* unhandled exception. */
if (ctx.channel().isOpen()) {
[11/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
TAJO-1397: Resource allocation should be fine grained. (jinho)
Closes #608
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5a02873d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5a02873d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5a02873d
Branch: refs/heads/master
Commit: 5a02873d5f0bd8fedd8527808bd3d4ecbe7d8af3
Parents: bedce3a
Author: Jinho Kim <jh...@apache.org>
Authored: Mon Jul 20 17:31:22 2015 +0900
Committer: Jinho Kim <jh...@apache.org>
Committed: Mon Jul 20 17:31:22 2015 +0900
----------------------------------------------------------------------
.travis.yml | 2 +-
CHANGES | 2 +
.../org/apache/tajo/catalog/FunctionDesc.java | 7 +-
.../org/apache/tajo/catalog/CatalogServer.java | 16 +-
.../dictionary/ClusterTableDescriptor.java | 6 +-
.../org/apache/tajo/cli/tools/TajoAdmin.java | 23 +-
tajo-client/src/main/proto/ClientProtos.proto | 19 +-
.../main/java/org/apache/tajo/SessionVars.java | 2 +-
.../java/org/apache/tajo/conf/TajoConf.java | 55 +-
.../java/org/apache/tajo/util/NumberUtil.java | 4 +
.../java/org/apache/tajo/util/StringUtils.java | 9 +-
.../main/java/org/apache/tajo/util/TUtil.java | 9 +
tajo-core/pom.xml | 13 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 16 +-
.../engine/planner/PhysicalPlannerImpl.java | 24 +-
.../tajo/engine/planner/enforce/Enforcer.java | 13 +-
.../tajo/engine/planner/global/DataChannel.java | 3 +-
.../tajo/engine/planner/global/MasterPlan.java | 11 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../tajo/engine/planner/physical/ScanExec.java | 10 +-
.../apache/tajo/engine/query/TaskRequest.java | 27 +-
.../tajo/engine/query/TaskRequestImpl.java | 87 +-
.../org/apache/tajo/master/ContainerProxy.java | 85 --
.../tajo/master/LaunchTaskRunnersEvent.java | 46 -
.../tajo/master/QueryCoordinatorService.java | 77 +-
.../org/apache/tajo/master/QueryInProgress.java | 102 ++-
.../org/apache/tajo/master/QueryManager.java | 104 +--
.../apache/tajo/master/TajoContainerProxy.java | 177 ----
.../java/org/apache/tajo/master/TajoMaster.java | 120 +--
.../tajo/master/TajoMasterClientService.java | 62 +-
.../tajo/master/TaskRunnerGroupEvent.java | 51 --
.../apache/tajo/master/TaskRunnerLauncher.java | 25 -
.../master/cluster/WorkerConnectionInfo.java | 4 +
.../tajo/master/container/TajoContainer.java | 173 ----
.../tajo/master/container/TajoContainerId.java | 171 ----
.../master/container/TajoContainerIdPBImpl.java | 100 ---
.../master/container/TajoConverterUtils.java | 87 --
.../master/event/ContainerAllocationEvent.java | 77 --
.../event/ContainerAllocatorEventType.java | 26 -
.../tajo/master/event/ContainerEvent.java | 37 -
.../event/GrouppedContainerAllocatorEvent.java | 45 -
.../tajo/master/event/LocalTaskEvent.java | 11 +-
.../tajo/master/event/QueryStartEvent.java | 9 +-
.../event/StageContainerAllocationEvent.java | 38 -
.../tajo/master/event/StageEventType.java | 2 -
.../master/event/StageShuffleReportEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 9 +-
.../tajo/master/event/TaskAttemptEventType.java | 1 +
.../event/TaskAttemptStatusUpdateEvent.java | 2 +-
.../event/TaskAttemptToSchedulerEvent.java | 26 +-
.../tajo/master/event/TaskCompletionEvent.java | 2 +-
.../tajo/master/event/TaskFatalErrorEvent.java | 2 +-
.../tajo/master/event/TaskRequestEvent.java | 35 +-
.../NonForwardQueryResultSystemScanner.java | 94 +--
.../org/apache/tajo/master/rm/NodeEvent.java | 37 +
.../apache/tajo/master/rm/NodeEventType.java | 30 +
.../tajo/master/rm/NodeLivelinessMonitor.java | 56 ++
.../tajo/master/rm/NodeReconnectEvent.java | 35 +
.../org/apache/tajo/master/rm/NodeState.java | 44 +
.../org/apache/tajo/master/rm/NodeStatus.java | 295 +++++++
.../apache/tajo/master/rm/NodeStatusEvent.java | 58 ++
.../apache/tajo/master/rm/TajoRMContext.java | 42 +-
.../tajo/master/rm/TajoResourceManager.java | 185 ++++
.../tajo/master/rm/TajoResourceTracker.java | 197 +++--
.../tajo/master/rm/TajoWorkerContainer.java | 125 ---
.../tajo/master/rm/TajoWorkerContainerId.java | 94 ---
.../master/rm/TajoWorkerResourceManager.java | 605 -------------
.../java/org/apache/tajo/master/rm/Worker.java | 290 -------
.../org/apache/tajo/master/rm/WorkerEvent.java | 37 -
.../apache/tajo/master/rm/WorkerEventType.java | 30 -
.../tajo/master/rm/WorkerLivelinessMonitor.java | 56 --
.../tajo/master/rm/WorkerReconnectEvent.java | 35 -
.../tajo/master/rm/WorkerResourceManager.java | 115 ---
.../org/apache/tajo/master/rm/WorkerState.java | 44 -
.../tajo/master/rm/WorkerStatusEvent.java | 54 --
.../scheduler/AbstractQueryScheduler.java | 73 ++
.../master/scheduler/QuerySchedulingInfo.java | 60 +-
.../apache/tajo/master/scheduler/QueueInfo.java | 101 +++
.../tajo/master/scheduler/QueueState.java | 50 ++
.../master/scheduler/SchedulingAlgorithms.java | 6 +-
.../master/scheduler/SimpleFifoScheduler.java | 148 ----
.../tajo/master/scheduler/SimpleScheduler.java | 388 +++++++++
.../master/scheduler/TajoResourceScheduler.java | 75 ++
.../event/ResourceReserveSchedulerEvent.java | 45 +
.../master/scheduler/event/SchedulerEvent.java | 27 +
.../scheduler/event/SchedulerEventType.java | 26 +
.../metrics/WorkerResourceMetricsGaugeSet.java | 16 +-
.../tajo/querymaster/AbstractTaskScheduler.java | 17 +-
.../tajo/querymaster/DefaultTaskScheduler.java | 500 ++++++-----
.../java/org/apache/tajo/querymaster/Query.java | 25 +-
.../apache/tajo/querymaster/QueryMaster.java | 249 +++---
.../querymaster/QueryMasterManagerService.java | 180 ++--
.../tajo/querymaster/QueryMasterTask.java | 218 +++--
.../apache/tajo/querymaster/Repartitioner.java | 4 +-
.../java/org/apache/tajo/querymaster/Stage.java | 247 ++----
.../java/org/apache/tajo/querymaster/Task.java | 29 +-
.../apache/tajo/querymaster/TaskAttempt.java | 34 +-
.../resource/DefaultResourceCalculator.java | 4 +-
.../org/apache/tajo/resource/NodeResource.java | 2 +-
.../java/org/apache/tajo/session/Session.java | 2 +-
.../main/java/org/apache/tajo/util/JSPUtil.java | 33 +-
.../apache/tajo/util/history/HistoryReader.java | 8 +-
.../apache/tajo/util/history/HistoryWriter.java | 78 +-
.../tajo/worker/AbstractResourceAllocator.java | 69 --
.../tajo/worker/ExecutionBlockContext.java | 70 +-
.../java/org/apache/tajo/worker/FetchImpl.java | 12 +-
.../org/apache/tajo/worker/LegacyTaskImpl.java | 844 -------------------
.../apache/tajo/worker/NodeResourceManager.java | 140 +--
.../apache/tajo/worker/NodeStatusUpdater.java | 120 ++-
.../apache/tajo/worker/ResourceAllocator.java | 29 -
.../tajo/worker/TajoResourceAllocator.java | 415 ---------
.../java/org/apache/tajo/worker/TajoWorker.java | 96 +--
.../tajo/worker/TajoWorkerManagerService.java | 79 +-
.../main/java/org/apache/tajo/worker/Task.java | 9 +-
.../org/apache/tajo/worker/TaskContainer.java | 1 -
.../org/apache/tajo/worker/TaskExecutor.java | 117 ++-
.../org/apache/tajo/worker/TaskHistory.java | 4 +-
.../java/org/apache/tajo/worker/TaskImpl.java | 31 +-
.../org/apache/tajo/worker/TaskManager.java | 161 +++-
.../java/org/apache/tajo/worker/TaskRunner.java | 306 -------
.../apache/tajo/worker/TaskRunnerHistory.java | 152 ----
.../apache/tajo/worker/TaskRunnerManager.java | 238 ------
.../tajo/worker/WorkerHeartbeatService.java | 262 ------
.../worker/event/ExecutionBlockErrorEvent.java | 41 +
.../worker/event/ExecutionBlockStartEvent.java | 35 -
.../worker/event/ExecutionBlockStopEvent.java | 16 +-
.../worker/event/NodeResourceAllocateEvent.java | 18 +-
.../event/NodeResourceDeallocateEvent.java | 8 +-
.../tajo/worker/event/NodeResourceEvent.java | 14 +-
.../worker/event/QMResourceAllocateEvent.java | 45 +
.../tajo/worker/event/QueryStopEvent.java | 35 +
.../tajo/worker/event/TaskExecutorEvent.java | 44 -
.../tajo/worker/event/TaskManagerEvent.java | 22 +-
.../tajo/worker/event/TaskRunnerEvent.java | 42 -
.../tajo/worker/event/TaskRunnerStartEvent.java | 39 -
.../tajo/worker/event/TaskRunnerStopEvent.java | 29 -
.../tajo/worker/event/TaskStartEvent.java | 18 +-
.../tajo/ws/rs/resources/ClusterResource.java | 8 +-
.../tajo/ws/rs/responses/WorkerResponse.java | 53 +-
.../src/main/proto/ContainerProtocol.proto | 48 --
.../main/proto/QueryCoordinatorProtocol.proto | 117 +--
.../src/main/proto/QueryMasterProtocol.proto | 11 +-
tajo-core/src/main/proto/ResourceProtos.proto | 311 +++++++
.../main/proto/ResourceTrackerProtocol.proto | 39 +-
.../src/main/proto/TajoWorkerProtocol.proto | 348 +-------
.../main/resources/webapps/admin/cluster.jsp | 99 +--
.../src/main/resources/webapps/admin/index.jsp | 34 +-
.../src/main/resources/webapps/admin/query.jsp | 57 +-
.../resources/webapps/admin/query_executor.jsp | 3 +-
.../resources/webapps/admin/querydetail.jsp | 4 +-
.../main/resources/webapps/admin/querytasks.jsp | 34 +-
.../src/main/resources/webapps/admin/task.jsp | 18 +-
.../src/main/resources/webapps/worker/index.jsp | 58 +-
.../resources/webapps/worker/querydetail.jsp | 21 +-
.../main/resources/webapps/worker/queryplan.jsp | 2 +-
.../resources/webapps/worker/querytasks.jsp | 44 +-
.../src/main/resources/webapps/worker/task.jsp | 16 +-
.../resources/webapps/worker/taskcontainers.jsp | 93 --
.../resources/webapps/worker/taskdetail.jsp | 39 +-
.../resources/webapps/worker/taskhistory.jsp | 6 +-
.../src/main/resources/webapps/worker/tasks.jsp | 107 ---
.../java/org/apache/tajo/QueryTestCaseBase.java | 2 +-
.../org/apache/tajo/TajoTestingCluster.java | 24 +-
.../planner/physical/TestBNLJoinExec.java | 2 +-
.../physical/TestFullOuterHashJoinExec.java | 2 +-
.../physical/TestFullOuterMergeJoinExec.java | 2 +-
.../planner/physical/TestHashJoinExec.java | 2 +-
.../physical/TestLeftOuterHashJoinExec.java | 2 +-
.../planner/physical/TestMergeJoinExec.java | 2 +-
.../planner/physical/TestPhysicalPlanner.java | 4 +-
.../physical/TestRightOuterMergeJoinExec.java | 2 +-
.../query/TestJoinOnPartitionedTables.java | 1 +
.../tajo/engine/query/TestOuterJoinQuery.java | 2 -
.../TestNonForwardQueryResultSystemScanner.java | 42 +-
.../apache/tajo/master/TestRepartitioner.java | 11 +-
.../tajo/master/rm/TestTajoResourceManager.java | 454 ----------
.../master/scheduler/TestFifoScheduler.java | 115 ---
.../master/scheduler/TestSimpleScheduler.java | 301 +++++++
.../apache/tajo/querymaster/TestKillQuery.java | 166 +---
.../apache/tajo/querymaster/TestQueryState.java | 38 +-
.../tajo/util/metrics/TestSystemMetrics.java | 1 +
.../apache/tajo/worker/MockExecutionBlock.java | 6 +-
.../tajo/worker/MockNodeResourceManager.java | 30 +-
.../tajo/worker/MockNodeStatusUpdater.java | 26 +-
.../apache/tajo/worker/MockTaskExecutor.java | 29 +-
.../org/apache/tajo/worker/MockTaskManager.java | 24 +-
.../apache/tajo/worker/MockWorkerContext.java | 20 -
.../org/apache/tajo/worker/TestFetcher.java | 14 +-
.../org/apache/tajo/worker/TestHistory.java | 124 ---
.../tajo/worker/TestNodeResourceManager.java | 100 ++-
.../tajo/worker/TestNodeStatusUpdater.java | 83 +-
.../apache/tajo/worker/TestTaskExecutor.java | 108 +--
.../org/apache/tajo/worker/TestTaskManager.java | 75 +-
.../configuration/worker_configuration.rst | 94 ++-
.../org/apache/tajo/plan/util/PlannerUtil.java | 8 +-
tajo-plan/src/main/proto/Plan.proto | 125 +++
tajo-project/pom.xml | 2 +-
.../org/apache/tajo/rpc/AsyncRpcServer.java | 6 +-
198 files changed, 5011 insertions(+), 9450 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index ef8fd97..671a31f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,7 +33,7 @@ notifications:
- issues@tajo.apache.org
irc: "chat.freenode.net#tajo"
-before_install: ulimit -t 514029
+before_install: ulimit -t 514029 -u 2048 -n 3000
install: ./dev-support/travis-install-dependencies.sh
script:
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 890aecf..1529c08 100644
--- a/CHANGES
+++ b/CHANGES
@@ -29,6 +29,8 @@ Release 0.11.0 - unreleased
IMPROVEMENT
+ TAJO-1397: Resource allocation should be fine grained. (jinho)
+
TAJO-1352: Improve the join order algorithm to consider missed cases of
associative join operators. (jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
index 6ea6ac6..9f71e8e 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java
@@ -38,7 +38,6 @@ import java.lang.reflect.Constructor;
*
*/
public class FunctionDesc implements ProtoObject<FunctionDescProto>, Cloneable, GsonObject, Comparable<FunctionDesc> {
- private FunctionDescProto.Builder builder = FunctionDescProto.newBuilder();
@Expose private FunctionSignature signature;
@Expose private FunctionInvocation invocation;
@@ -184,11 +183,7 @@ public class FunctionDesc implements ProtoObject<FunctionDescProto>, Cloneable,
@Override
public FunctionDescProto getProto() {
- if (builder == null) {
- builder = FunctionDescProto.newBuilder();
- } else {
- builder.clear();
- }
+ FunctionDescProto.Builder builder = FunctionDescProto.newBuilder();
builder.setSignature(signature.getProto());
builder.setSupplement(supplement.getProto());
builder.setInvocation(invocation.getProto());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index f2e9795..b1410dd 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -172,7 +172,8 @@ public class CatalogServer extends AbstractService {
}
}
- public void start() {
+ @Override
+ public void serviceStart() throws Exception {
String serverAddr = conf.getVar(ConfVars.CATALOG_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(serverAddr);
int workerNum = conf.getIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM);
@@ -189,10 +190,11 @@ public class CatalogServer extends AbstractService {
}
LOG.info("Catalog Server startup (" + bindAddressStr + ")");
- super.start();
+ super.serviceStart();
}
- public void stop() {
+ @Override
+ public void serviceStop() throws Exception {
LOG.info("Catalog Server (" + bindAddressStr + ") shutdown");
// If CatalogServer shutdowns before it started, rpcServer and store may be NULL.
@@ -201,13 +203,9 @@ public class CatalogServer extends AbstractService {
this.rpcServer.shutdown();
}
if (store != null) {
- try {
- store.close();
- } catch (IOException ioe) {
- LOG.error(ioe.getMessage(), ioe);
- }
+ store.close();
}
- super.stop();
+ super.serviceStop();
}
public CatalogProtocolHandler getHandler() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ClusterTableDescriptor.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ClusterTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ClusterTableDescriptor.java
index e3c830f..69067f4 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ClusterTableDescriptor.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/ClusterTableDescriptor.java
@@ -31,10 +31,8 @@ class ClusterTableDescriptor extends AbstractTableDescriptor {
new ColumnDescriptor("total_cpu", Type.INT4, 0),
new ColumnDescriptor("used_mem", Type.INT8, 0),
new ColumnDescriptor("total_mem", Type.INT8, 0),
- new ColumnDescriptor("free_heap", Type.INT8, 0),
- new ColumnDescriptor("max_heap", Type.INT8, 0),
- new ColumnDescriptor("used_diskslots", Type.FLOAT4, 0),
- new ColumnDescriptor("total_diskslots", Type.FLOAT4, 0),
+ new ColumnDescriptor("used_disk", Type.INT4, 0),
+ new ColumnDescriptor("total_disk", Type.INT4, 0),
new ColumnDescriptor("running_tasks", Type.INT4, 0),
new ColumnDescriptor("last_heartbeat_ts", Type.TIMESTAMP, 0)
};
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
index 739cd54..76ba7a9 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
@@ -268,7 +268,7 @@ public class TajoAdmin {
} else {
String fmtQueryMasterLine = "%1$-25s %2$-5s %3$-5s %4$-10s %5$-10s%n";
line = String.format(fmtQueryMasterLine, "QueryMaster", "Port", "Query",
- "Heap", "Status");
+ "Mem", "Status");
writer.write(line);
line = String.format(fmtQueryMasterLine, DASHLINE_LEN25, DASHLINE_LEN5,
DASHLINE_LEN5, DASHLINE_LEN10, DASHLINE_LEN10);
@@ -276,12 +276,12 @@ public class TajoAdmin {
for (WorkerResourceInfo queryMaster : liveQueryMasters) {
TajoProtos.WorkerConnectionInfoProto connInfo = queryMaster.getConnectionInfo();
String queryMasterHost = String.format("%s:%d", connInfo.getHost(), connInfo.getQueryMasterPort());
- String heap = String.format("%d MB", queryMaster.getMaxHeap() / 1024 / 1024);
+ String memory = String.format("%d MB", queryMaster.getAvailableResource().getMemory());
line = String.format(fmtQueryMasterLine,
queryMasterHost,
connInfo.getClientPort(),
queryMaster.getNumQueryMasterTasks(),
- heap,
+ memory,
queryMaster.getWorkerStatus());
writer.write(line);
}
@@ -348,7 +348,7 @@ public class TajoAdmin {
String line = String.format(fmtWorkerLine,
"Worker", "Port", "Tasks",
"Mem", "Disk",
- "Heap", "Status");
+ "Cpu", "Status");
writer.write(line);
line = String.format(fmtWorkerLine,
DASHLINE_LEN25, DASHLINE_LEN5, DASHLINE_LEN5,
@@ -359,17 +359,16 @@ public class TajoAdmin {
for (WorkerResourceInfo worker : workers) {
TajoProtos.WorkerConnectionInfoProto connInfo = worker.getConnectionInfo();
String workerHost = String.format("%s:%d", connInfo.getHost(), connInfo.getPeerRpcPort());
- String mem = String.format("%d/%d", worker.getUsedMemoryMB(),
- worker.getMemoryMB());
- String disk = String.format("%.2f/%.2f", worker.getUsedDiskSlots(),
- worker.getDiskSlots());
- String heap = String.format("%d/%d MB", worker.getFreeHeap()/1024/1024,
- worker.getMaxHeap()/1024/1024);
-
+ String mem = String.format("%d/%d", worker.getAvailableResource().getMemory(),
+ worker.getTotalResource().getMemory());
+ String disk = String.format("%d/%d", worker.getAvailableResource().getDisks(),
+ worker.getTotalResource().getDisks());
+ String cpu = String.format("%d/%d", worker.getAvailableResource().getVirtualCores(),
+ worker.getTotalResource().getVirtualCores());
line = String.format(fmtWorkerLine, workerHost,
connInfo.getPullServerPort(),
worker.getNumRunningTasks(),
- mem, disk, heap, worker.getWorkerStatus());
+ mem, disk, cpu, worker.getWorkerStatus());
writer.write(line);
}
writer.write("\n\n");
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-client/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto
index 5497faa..9c20fd8 100644
--- a/tajo-client/src/main/proto/ClientProtos.proto
+++ b/tajo-client/src/main/proto/ClientProtos.proto
@@ -177,19 +177,12 @@ message GetClusterInfoRequest {
message WorkerResourceInfo {
required WorkerConnectionInfoProto connectionInfo = 1;
- required float diskSlots = 2;
- required int32 cpuCoreSlots = 3;
- required int32 memoryMB = 4;
- required float usedDiskSlots = 5;
- required int32 usedMemoryMB = 6;
- required int32 usedCpuCoreSlots = 7;
- required int64 maxHeap = 8;
- required int64 freeHeap = 9;
- required int64 totalHeap = 10;
- required int32 numRunningTasks = 11;
- required string workerStatus = 12;
- required int64 lastHeartbeat = 13;
- required int32 numQueryMasterTasks = 14;
+ required NodeResourceProto totalResource = 2;
+ required NodeResourceProto availableResource = 3;
+ required int32 numRunningTasks = 4;
+ required string workerStatus = 5;
+ required int64 lastHeartbeat = 6;
+ required int32 numQueryMasterTasks = 7;
}
message GetClusterInfoResponse {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
index 28fdb0b..832a5b4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@ -108,7 +108,7 @@ public enum SessionVars implements ConfigKey {
Boolean.class, Validators.bool()),
QUERY_EXECUTE_PARALLEL(ConfVars.$QUERY_EXECUTE_PARALLEL_MAX, "Maximum parallel running of execution blocks for a query",
- DEFAULT, Integer.class, Validators.min("0")),
+ DEFAULT, Integer.class, Validators.min("1")),
// for physical Executors
EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT,
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index 14cfb11..0436116 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -152,12 +152,13 @@ public class TajoConf extends Configuration {
// Resource tracker service
RESOURCE_TRACKER_RPC_ADDRESS("tajo.resource-tracker.rpc.address", "localhost:26003",
Validators.networkAddr()),
- RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120 * 1000), // seconds
+ RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120), // seconds
// QueryMaster resource
- TAJO_QUERYMASTER_DISK_SLOT("tajo.qm.resource.disk.slots", 0.0f, Validators.min("0.0f")),
- TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512, Validators.min("64")),
- TAJO_QUERYMASTER_ALLOCATION_TIMEOUT("tajo.qm.resource.allocation.timeout", "3 sec"),
+ QUERYMASTER_MINIMUM_MEMORY("tajo.qm.resource.min.memory-mb", 500, Validators.min("64")),
+
+ // Worker task resource
+ TASK_RESOURCE_MINIMUM_MEMORY("tajo.task.resource.min.memory-mb", 500, Validators.min("64")),
// Tajo Worker Service Addresses
WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080", Validators.networkAddr()),
@@ -172,42 +173,32 @@ public class TajoConf extends Configuration {
// Tajo Worker Resources
WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores",
- Runtime.getRuntime().availableProcessors(), Validators.min("1")),
- WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024, Validators.min("64")),
- @Deprecated
- WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1.0f),
- WORKER_RESOURCE_AVAILABLE_DISKS_NUM("tajo.worker.resource.disks.num", 1, Validators.min("1")),
+ Runtime.getRuntime().availableProcessors(), Validators.min("2")), // 1qm + 1task
+ WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1000, Validators.min("64")),
+
+ WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1, Validators.min("1")),
+
WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM("tajo.worker.resource.disk.parallel-execution.num", 2,
Validators.min("1")),
- WORKER_EXECUTION_MAX_SLOTS("tajo.worker.parallel-execution.max-num", 2),
- WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()),
- // Tajo Worker Dedicated Resources
- WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false, Validators.bool()),
- WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f,
- Validators.range("0.0f", "1.0f")),
+ WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()),
// Tajo History
WORKER_HISTORY_EXPIRE_PERIOD("tajo.worker.history.expire-interval-minutes", 60), // 1 hours
- QUERYMASTER_HISTORY_EXPIRE_PERIOD("tajo.qm.history.expire-interval-minutes", 6 * 60), // 6 hours
+ QUERYMASTER_CACHE_EXPIRE_PERIOD("tajo.qm.history.expire-interval-minutes", 10), // 10 mins
- WORKER_HEARTBEAT_INTERVAL("tajo.worker.heartbeat.interval", 10 * 1000), // 10 sec
+ WORKER_HEARTBEAT_IDLE_INTERVAL("tajo.worker.heartbeat.idle.interval", 10 * 1000), // 10 sec
+ WORKER_HEARTBEAT_ACTIVE_INTERVAL("tajo.worker.heartbeat.active.interval", 1000), // 1 sec
- // Resource Manager
- RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager",
+ //Default query scheduler
+ RESOURCE_SCHEDULER_CLASS("tajo.resource.scheduler", "org.apache.tajo.master.scheduler.SimpleScheduler",
Validators.groups(Validators.notNull(), Validators.clazz())),
+ QUERYMASTER_TASK_SCHEDULER_DELAY("tajo.qm.task-scheduler.delay", 50), // 50 ms
+
// Catalog
CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005", Validators.networkAddr()),
-
- // for Yarn Resource Manager ----------------------------------------------
-
- /** how many launching TaskRunners in parallel */
- @Deprecated
- YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num",
- Runtime.getRuntime().availableProcessors() * 2),
-
// Query Configuration
QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60, Validators.min("0")),
QUERY_SESSION_QUERY_CACHE_SIZE("tajo.query.session.query-cache-size-kb", 1024, Validators.min("0")),
@@ -216,7 +207,8 @@ public class TajoConf extends Configuration {
PULLSERVER_PORT("tajo.pullserver.port", 0, Validators.range("0", "65535")),
SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false, Validators.bool()),
SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"),
- SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2),
+ SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num",
+ Runtime.getRuntime().availableProcessors() * 2, Validators.min("1")),
SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192),
SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120),
SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20),
@@ -262,8 +254,6 @@ public class TajoConf extends Configuration {
Runtime.getRuntime().availableProcessors() * 1),
// Task Configuration -----------------------------------------------------
- TASK_DEFAULT_MEMORY("tajo.task.memory-slot-mb.default", 512),
- TASK_DEFAULT_DISK("tajo.task.disk-slot.default", 0.5f),
TASK_DEFAULT_SIZE("tajo.task.size-mb", 128),
// Query and Optimization -------------------------------------------------
@@ -285,8 +275,6 @@ public class TajoConf extends Configuration {
HISTORY_QUERY_DIR("tajo.history.query.dir", STAGING_ROOT_DIR.defaultVal + "/history"),
HISTORY_TASK_DIR("tajo.history.task.dir", "file:///tmp/tajo-${user.name}/history"),
HISTORY_EXPIRY_TIME_DAY("tajo.history.expiry-time-day", 7),
- HISTORY_QUERY_REPLICATION("tajo.history.query.replication", 1, Validators.min("1")),
- HISTORY_TASK_REPLICATION("tajo.history.task.replication", 1, Validators.min("1")),
// Misc -------------------------------------------------------------------
// Fragment
@@ -329,8 +317,7 @@ public class TajoConf extends Configuration {
$GROUPBY_MULTI_LEVEL_ENABLED("tajo.dist-query.groupby.multi-level-aggr", true),
- // WARN "tajo.yarn-rm.parallel-task-runner-launcher-num" should be set enough to avoid deadlock
- $QUERY_EXECUTE_PARALLEL_MAX("tajo.query.execute.parallel.max", 1),
+ $QUERY_EXECUTE_PARALLEL_MAX("tajo.query.execute.parallel.max", 10),
// for physical Executors
$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L),
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
index 0d70cc2..32e086c 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java
@@ -1050,4 +1050,8 @@ public class NumberUtil {
return returnNumber;
}
+
+ public static int compare(long x, long y) {
+ return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 0a16072..018c62a 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -18,6 +18,7 @@
package org.apache.tajo.util;
+import io.netty.util.CharsetUtil;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.SystemUtils;
@@ -77,10 +78,12 @@ public class StringUtils {
}
return buf.toString();
}
-
- static CharsetEncoder asciiEncoder = Charset.forName("US-ASCII").newEncoder(); // or "ISO-8859-1" for ISO Latin 1
-
+ /**
+ * Check Seven-bit ASCII
+ */
public static boolean isPureAscii(String v) {
+ // get thread-safe encoder
+ CharsetEncoder asciiEncoder = CharsetUtil.getEncoder(CharsetUtil.US_ASCII);
return asciiEncoder.canEncode(v);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
index 2293ef5..66e8acc 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
@@ -267,4 +267,13 @@ public class TUtil {
StackTraceElement element = ste[2 + depth];
return element.getClassName() + ":" + element.getMethodName() + "(" + element.getLineNumber() +")";
}
+
+ public static <T> T checkTypeAndGet(Object instance, Class<T> type) {
+ if (!type.isInstance(instance)) {
+ throw new IllegalArgumentException(instance.getClass().getSimpleName()
+ + " must be a " + type.getSimpleName() + " type");
+
+ }
+ return (T) instance;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index 33e37b6..bd52e12 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -163,12 +163,12 @@
<argument>--proto_path=../tajo-client/src/main/proto</argument>
<argument>--proto_path=../tajo-plan/src/main/proto</argument>
<argument>--java_out=target/generated-sources/proto</argument>
- <argument>src/main/proto/ContainerProtocol.proto</argument>
<argument>src/main/proto/ResourceTrackerProtocol.proto</argument>
<argument>src/main/proto/QueryMasterProtocol.proto</argument>
<argument>src/main/proto/QueryCoordinatorProtocol.proto</argument>
<argument>src/main/proto/TajoWorkerProtocol.proto</argument>
<argument>src/main/proto/InternalTypes.proto</argument>
+ <argument>src/main/proto/ResourceProtos.proto</argument>
</arguments>
</configuration>
<goals>
@@ -390,11 +390,6 @@
<version>1.2.3</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jdt</groupId>
- <artifactId>core</artifactId>
- <version>3.1.1</version>
- </dependency>
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
</dependency>
@@ -407,6 +402,12 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-2.1</artifactId>
<version>6.1.14</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>core</artifactId>
+ <groupId>org.eclipse.jdt</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 9f850f9..5c62654 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -28,9 +28,7 @@ import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.Aggregation.GroupType;
import org.apache.tajo.algebra.LiteralValue.LiteralType;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.engine.parser.SQLParser.*;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.StringUtils;
@@ -44,7 +42,6 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
import static org.apache.tajo.engine.parser.SQLParser.*;
public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
- private SQLParser parser;
public SQLAnalyzer() {
}
@@ -53,15 +50,14 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
ANTLRInputStream input = new ANTLRInputStream(sql);
SQLLexer lexer = new SQLLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
- this.parser = new SQLParser(tokens);
- parser.setBuildParseTree(true);
- parser.removeErrorListeners();
-
- parser.setErrorHandler(new SQLErrorStrategy());
- parser.addErrorListener(new SQLErrorListener());
-
SqlContext context;
try {
+ SQLParser parser = new SQLParser(tokens);
+ parser.setBuildParseTree(true);
+ parser.removeErrorListeners();
+
+ parser.setErrorHandler(new SQLErrorStrategy());
+ parser.addErrorListener(new SQLErrorListener());
context = parser.sql();
} catch (SQLParseError e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index c6b9b41..377aebe 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -40,11 +40,10 @@ import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.physical.*;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.InternalException;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
@@ -66,12 +65,13 @@ import java.util.Stack;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.GroupbyEnforce.GroupbyAlgorithm;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.SortEnforce;
+import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
+import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty.EnforceType;
+import static org.apache.tajo.plan.serder.PlanProto.GroupbyEnforce.GroupbyAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.SortedInputEnforce;
+import static org.apache.tajo.plan.serder.PlanProto.SortEnforce;
public class PhysicalPlannerImpl implements PhysicalPlanner {
private static final Log LOG = LogFactory.getLog(PhysicalPlannerImpl.class);
@@ -885,7 +885,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
List<EnforceProperty> property = enforcer.getEnforceProperties(EnforceType.SORTED_INPUT);
if (property != null && property.size() > 0 && node.peek().getType() == NodeType.SORT) {
SortNode sortNode = (SortNode) node.peek();
- TajoWorkerProtocol.SortedInputEnforce sortEnforcer = property.get(0).getSortedInput();
+ SortedInputEnforce sortEnforcer = property.get(0).getSortedInput();
boolean condition = scanNode.getTableName().equals(sortEnforcer.getTableName());
SortSpec [] sortSpecs = LogicalNodeDeserializer.convertSortSpecs(sortEnforcer.getSortSpecsList());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java
index 8128390..92ecadd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/enforce/Enforcer.java
@@ -18,24 +18,21 @@
package org.apache.tajo.engine.planner.enforce;
-
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.util.TUtil;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.GroupbyEnforce.GroupbyAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.*;
+import static org.apache.tajo.plan.serder.PlanProto.GroupbyEnforce.GroupbyAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.*;
+import static org.apache.tajo.plan.serder.PlanProto.EnforceProperty.EnforceType;
+import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
public class Enforcer implements ProtoObject<EnforcerProto> {
Map<EnforceType, List<EnforceProperty>> properties;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
index 3adc0a3..e09684a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
@@ -25,8 +25,7 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.util.StringUtils;
-import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
+import static org.apache.tajo.plan.serder.PlanProto.DataChannelProto;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
import static org.apache.tajo.plan.serder.PlanProto.TransmitType;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java
index f8cd1e9..80317b0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java
@@ -23,13 +23,14 @@ package org.apache.tajo.engine.planner.global;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
+import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
+import org.apache.tajo.plan.serder.PlanProto.ShuffleType;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.util.graph.DirectedGraphVisitor;
import org.apache.tajo.util.graph.SimpleDirectedGraph;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import java.util.ArrayList;
import java.util.HashMap;
@@ -37,8 +38,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
-
public class MasterPlan {
private final QueryId queryId;
private final QueryContext context;
@@ -288,7 +287,7 @@ public class MasterPlan {
if (block.getEnforcer().getProperties().size() > 0) {
sb.append("\n[Enforcers]\n");
int i = 0;
- for (TajoWorkerProtocol.EnforceProperty enforce : block.getEnforcer().getProperties()) {
+ for (EnforceProperty enforce : block.getEnforcer().getProperties()) {
sb.append(" ").append(i++).append(": ");
sb.append(Enforcer.toString(enforce));
sb.append("\n");
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
index 8095458..f181193 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
@@ -30,9 +30,9 @@ import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.planner.global.GlobalPlanner.GlobalPlanContext;
import org.apache.tajo.engine.planner.global.MasterPlan;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.PlanningException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ScanExec.java
index 86874ba..5cca4c5 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ScanExec.java
@@ -21,7 +21,7 @@ package org.apache.tajo.engine.planner.physical;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.engine.planner.enforce.Enforcer;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
@@ -57,11 +57,11 @@ public abstract class ScanExec extends PhysicalExec {
private boolean checkIfBroadcast() {
Enforcer enforcer = context.getEnforcer();
- if (enforcer != null && enforcer.hasEnforceProperty(TajoWorkerProtocol.EnforceProperty.EnforceType.BROADCAST)) {
- List<TajoWorkerProtocol.EnforceProperty> properties =
- enforcer.getEnforceProperties(TajoWorkerProtocol.EnforceProperty.EnforceType.BROADCAST);
+ if (enforcer != null && enforcer.hasEnforceProperty(EnforceProperty.EnforceType.BROADCAST)) {
+ List<EnforceProperty> properties =
+ enforcer.getEnforceProperties(EnforceProperty.EnforceType.BROADCAST);
- for (TajoWorkerProtocol.EnforceProperty property : properties) {
+ for (EnforceProperty property : properties) {
if (getCanonicalName().equals(property.getBroadcast().getTableName())) {
return true;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
index 2fa272a..48d4780 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
@@ -21,32 +21,27 @@
*/
package org.apache.tajo.engine.query;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.worker.FetchImpl;
import java.util.List;
-public interface TaskRequest extends ProtoObject<TajoWorkerProtocol.TaskRequestProto> {
+public interface TaskRequest extends ProtoObject<TaskRequestProto> {
- public TaskAttemptId getId();
- public List<CatalogProtos.FragmentProto> getFragments();
- public String getOutputTableId();
- public boolean isClusteredOutput();
- public PlanProto.LogicalNodeTree getPlan();
- public boolean isInterQuery();
- public void setInterQuery();
- public void addFetch(String name, FetchImpl fetch);
- public List<FetchImpl> getFetches();
- public boolean shouldDie();
- public void setShouldDie();
- public QueryContext getQueryContext(TajoConf conf);
- public DataChannel getDataChannel();
- public Enforcer getEnforcer();
+ TaskAttemptId getId();
+ List<CatalogProtos.FragmentProto> getFragments();
+ PlanProto.LogicalNodeTree getPlan();
+ void setInterQuery();
+ void addFetch(String name, FetchImpl fetch);
+ List<FetchImpl> getFetches();
+ QueryContext getQueryContext(TajoConf conf);
+ DataChannel getDataChannel();
+ Enforcer getEnforcer();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
index b4727dc..f97d005 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
@@ -22,9 +22,9 @@ import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProtoOrBuilder;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
+import org.apache.tajo.ResourceProtos.FetchProto;
+import org.apache.tajo.ResourceProtos.TaskRequestProtoOrBuilder;
import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.worker.FetchImpl;
@@ -34,22 +34,22 @@ import java.util.List;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
public class TaskRequestImpl implements TaskRequest {
-
- private TaskAttemptId id;
- private List<FragmentProto> fragments;
- private String outputTable;
+
+ private TaskAttemptId id;
+ private List<FragmentProto> fragments;
+ private String outputTable;
private boolean isUpdated;
private boolean clusteredOutput;
private PlanProto.LogicalNodeTree plan; // logical node
private Boolean interQuery;
private List<FetchImpl> fetches;
- private Boolean shouldDie;
- private QueryContext queryContext;
- private DataChannel dataChannel;
- private Enforcer enforcer;
+ private QueryContext queryContext;
+ private DataChannel dataChannel;
+ private Enforcer enforcer;
+ private String queryMasterHostAndPort;
- private TaskRequestProto proto = TajoWorkerProtocol.TaskRequestProto.getDefaultInstance();
- private TajoWorkerProtocol.TaskRequestProto.Builder builder = null;
+ private TaskRequestProto proto = TaskRequestProto.getDefaultInstance();
+ private TaskRequestProto.Builder builder = null;
private boolean viaProto = false;
public TaskRequestImpl() {
@@ -61,9 +61,9 @@ public class TaskRequestImpl implements TaskRequest {
public TaskRequestImpl(TaskAttemptId id, List<FragmentProto> fragments,
String outputTable, boolean clusteredOutput,
PlanProto.LogicalNodeTree plan, QueryContext queryContext, DataChannel channel,
- Enforcer enforcer) {
+ Enforcer enforcer, String queryMasterHostAndPort) {
this();
- this.set(id, fragments, outputTable, clusteredOutput, plan, queryContext, channel, enforcer);
+ this.set(id, fragments, outputTable, clusteredOutput, plan, queryContext, channel, enforcer, queryMasterHostAndPort);
}
public TaskRequestImpl(TaskRequestProto proto) {
@@ -75,7 +75,8 @@ public class TaskRequestImpl implements TaskRequest {
public void set(TaskAttemptId id, List<FragmentProto> fragments,
String outputTable, boolean clusteredOutput,
- PlanProto.LogicalNodeTree plan, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) {
+ PlanProto.LogicalNodeTree plan, QueryContext queryContext,
+ DataChannel dataChannel, Enforcer enforcer, String queryMasterHostAndPort) {
this.id = id;
this.fragments = fragments;
this.outputTable = outputTable;
@@ -86,6 +87,7 @@ public class TaskRequestImpl implements TaskRequest {
this.queryContext = queryContext;
this.dataChannel = dataChannel;
this.enforcer = enforcer;
+ this.queryMasterHostAndPort = queryMasterHostAndPort;
}
@Override
@@ -124,32 +126,6 @@ public class TaskRequestImpl implements TaskRequest {
return this.fragments;
}
- @Override
- public String getOutputTableId() {
- TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (outputTable != null) {
- return this.outputTable;
- }
- if (!p.hasOutputTable()) {
- return null;
- }
- this.outputTable = p.getOutputTable();
- return this.outputTable;
- }
-
- @Override
- public boolean isClusteredOutput() {
- TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (isUpdated) {
- return this.clusteredOutput;
- }
- if (!p.hasClusteredOutput()) {
- return false;
- }
- this.clusteredOutput = p.getClusteredOutput();
- this.isUpdated = true;
- return this.clusteredOutput;
- }
@Override
public PlanProto.LogicalNodeTree getPlan() {
@@ -248,33 +224,14 @@ public class TaskRequestImpl implements TaskRequest {
}
TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
this.fetches = new ArrayList<FetchImpl>();
- for(TajoWorkerProtocol.FetchProto fetch : p.getFetchesList()) {
+ for(FetchProto fetch : p.getFetchesList()) {
fetches.add(new FetchImpl(fetch));
}
}
- @Override
- public boolean shouldDie() {
- TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (shouldDie != null) {
- return shouldDie;
- }
- if (!p.hasShouldDie()) {
- return false;
- }
- this.shouldDie = p.getShouldDie();
- return this.shouldDie;
- }
-
- @Override
- public void setShouldDie() {
- maybeInitBuilder();
- shouldDie = true;
- }
-
private void maybeInitBuilder() {
if (viaProto || builder == null) {
- builder = TajoWorkerProtocol.TaskRequestProto.newBuilder(proto);
+ builder = TaskRequestProto.newBuilder(proto);
}
viaProto = true;
}
@@ -305,8 +262,8 @@ public class TaskRequestImpl implements TaskRequest {
builder.addFetches(fetches.get(i).getProto());
}
}
- if (this.shouldDie != null) {
- builder.setShouldDie(this.shouldDie);
+ if (this.queryMasterHostAndPort != null) {
+ builder.setQueryMasterHostAndPort(this.queryMasterHostAndPort);
}
if (this.queryContext != null) {
builder.setQueryContext(queryContext.getProto());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
deleted file mode 100644
index cad63a0..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.querymaster.QueryMasterTask;
-import org.apache.tajo.master.container.TajoContainer;
-import org.apache.tajo.master.container.TajoContainerId;
-
-public abstract class ContainerProxy {
- protected static final Log LOG = LogFactory.getLog(ContainerProxy.class);
-
- final public static FsPermission QUERYCONF_FILE_PERMISSION =
- FsPermission.createImmutable((short) 0644); // rw-r--r--
-
-
- protected static enum ContainerState {
- PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
- }
-
- protected final ExecutionBlockId executionBlockId;
- protected Configuration conf;
- protected QueryMasterTask.QueryMasterTaskContext context;
-
- protected ContainerState state;
- // store enough information to be able to cleanup the container
- protected TajoContainer container;
- protected TajoContainerId containerId;
- protected String hostName;
- protected int port = -1;
-
- public abstract void launch(ContainerLaunchContext containerLaunchContext);
- public abstract void stopContainer();
-
- public ContainerProxy(QueryMasterTask.QueryMasterTaskContext context, Configuration conf,
- ExecutionBlockId executionBlockId, TajoContainer container) {
- this.context = context;
- this.conf = conf;
- this.state = ContainerState.PREP;
- this.container = container;
- this.executionBlockId = executionBlockId;
- this.containerId = container.getId();
- }
-
- public synchronized boolean isCompletelyDone() {
- return state == ContainerState.DONE || state == ContainerState.FAILED;
- }
-
- public String getTaskHostName() {
- return this.hostName;
- }
-
- public int getTaskPort() {
- return this.port;
- }
-
- public TajoContainerId getContainerId() {
- return containerId;
- }
-
- public ExecutionBlockId getBlockId() {
- return executionBlockId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
deleted file mode 100644
index e620afa..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master;
-
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.container.TajoContainer;
-
-import java.util.Collection;
-
-public class LaunchTaskRunnersEvent extends TaskRunnerGroupEvent {
- private final QueryContext queryContext;
- private final String planJson;
-
- public LaunchTaskRunnersEvent(ExecutionBlockId executionBlockId,
- Collection<TajoContainer> containers, QueryContext queryContext,
- String planJson) {
- super(EventType.CONTAINER_REMOTE_LAUNCH, executionBlockId, containers);
- this.queryContext = queryContext;
- this.planJson = planJson;
- }
-
- public QueryContext getQueryContext() {
- return queryContext;
- }
-
- public String getPlanJson() {
- return planJson;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java
index 1b1d49e..3b04fc5 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java
@@ -23,13 +23,13 @@ import com.google.protobuf.RpcController;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.QueryCoordinatorProtocol;
import org.apache.tajo.ipc.QueryCoordinatorProtocol.*;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
+import org.apache.tajo.master.rm.NodeStatus;
+import org.apache.tajo.master.scheduler.event.ResourceReserveSchedulerEvent;
import org.apache.tajo.rpc.AsyncRpcServer;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto;
@@ -37,7 +37,8 @@ import org.apache.tajo.util.NetUtils;
import java.net.InetSocketAddress;
import java.util.Collection;
-import java.util.List;
+
+import static org.apache.tajo.ResourceProtos.*;
public class QueryCoordinatorService extends AbstractService {
private final static Log LOG = LogFactory.getLog(QueryCoordinatorService.class);
@@ -59,30 +60,27 @@ public class QueryCoordinatorService extends AbstractService {
}
@Override
- public void start() {
+ public void serviceStart() throws Exception {
String confMasterServiceAddr = conf.getVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(confMasterServiceAddr);
int workerNum = conf.getIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM);
- try {
- server = new AsyncRpcServer(QueryCoordinatorProtocol.class, masterHandler, initIsa, workerNum);
- } catch (Exception e) {
- LOG.error(e, e);
- }
+
+ server = new AsyncRpcServer(QueryCoordinatorProtocol.class, masterHandler, initIsa, workerNum);
server.start();
bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
this.conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
NetUtils.normalizeInetSocketAddress(bindAddress));
LOG.info("Instantiated TajoMasterService at " + this.bindAddress);
- super.start();
+ super.serviceStart();
}
@Override
- public void stop() {
+ public void serviceStop() throws Exception {
if(server != null) {
server.shutdown();
server = null;
}
- super.stop();
+ super.serviceStop();
}
public InetSocketAddress getBindAddress() {
@@ -97,62 +95,47 @@ public class QueryCoordinatorService extends AbstractService {
@Override
public void heartbeat(
RpcController controller,
- TajoHeartbeat request, RpcCallback<QueryCoordinatorProtocol.TajoHeartbeatResponse> done) {
+ TajoHeartbeatRequest request, RpcCallback<TajoHeartbeatResponse> done) {
if(LOG.isDebugEnabled()) {
LOG.debug("Received QueryHeartbeat:" + new WorkerConnectionInfo(request.getConnectionInfo()));
}
- QueryCoordinatorProtocol.TajoHeartbeatResponse.ResponseCommand command = null;
+ TajoHeartbeatResponse.ResponseCommand command;
QueryManager queryManager = context.getQueryJobManager();
command = queryManager.queryHeartbeat(request);
- QueryCoordinatorProtocol.TajoHeartbeatResponse.Builder builder = QueryCoordinatorProtocol.TajoHeartbeatResponse.newBuilder();
+ TajoHeartbeatResponse.Builder builder = TajoHeartbeatResponse.newBuilder();
builder.setHeartbeatResult(BOOL_TRUE);
if(command != null) {
builder.setResponseCommand(command);
}
- builder.setClusterResourceSummary(context.getResourceManager().getClusterResourceSummary());
done.run(builder.build());
}
+ /**
+ * Reserve a node resources to TajoMaster
+ */
@Override
- public void allocateWorkerResources(
- RpcController controller,
- QueryCoordinatorProtocol.WorkerResourceAllocationRequest request,
- RpcCallback<QueryCoordinatorProtocol.WorkerResourceAllocationResponse> done) {
- context.getResourceManager().allocateWorkerResources(request, done);
- }
-
- @Override
- public void releaseWorkerResource(RpcController controller, WorkerResourceReleaseRequest request,
- RpcCallback<PrimitiveProtos.BoolProto> done) {
- List<ContainerProtocol.TajoContainerIdProto> containerIds = request.getContainerIdsList();
-
- for(ContainerProtocol.TajoContainerIdProto eachContainer: containerIds) {
- context.getResourceManager().releaseWorkerResource(eachContainer);
- }
- done.run(BOOL_TRUE);
+ public void reserveNodeResources(RpcController controller, NodeResourceRequest request,
+ RpcCallback<NodeResourceResponse> done) {
+ Dispatcher dispatcher = context.getResourceManager().getRMContext().getDispatcher();
+ dispatcher.getEventHandler().handle(new ResourceReserveSchedulerEvent(request, done));
}
+ /**
+ * Get all worker connection information
+ */
@Override
- public void getAllWorkerResource(RpcController controller, PrimitiveProtos.NullProto request,
- RpcCallback<WorkerResourcesRequest> done) {
-
- WorkerResourcesRequest.Builder builder = WorkerResourcesRequest.newBuilder();
- Collection<Worker> workers = context.getResourceManager().getWorkers().values();
-
- for(Worker worker: workers) {
- WorkerResource resource = worker.getResource();
-
- WorkerResourceProto.Builder workerResource = WorkerResourceProto.newBuilder();
+ public void getAllWorkers(RpcController controller, PrimitiveProtos.NullProto request,
+ RpcCallback<WorkerConnectionsResponse> done) {
- workerResource.setConnectionInfo(worker.getConnectionInfo().getProto());
- workerResource.setMemoryMB(resource.getMemoryMB());
- workerResource.setDiskSlots(resource.getDiskSlots());
+ WorkerConnectionsResponse.Builder builder = WorkerConnectionsResponse.newBuilder();
+ Collection<NodeStatus> nodeStatuses = context.getResourceManager().getRMContext().getNodes().values();
- builder.addWorkerResources(workerResource);
+ for(NodeStatus nodeStatus : nodeStatuses) {
+ builder.addWorker(nodeStatus.getConnectionInfo().getProto());
}
done.run(builder.build());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
index ece42f7..e22663a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
@@ -22,24 +22,26 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.tajo.QueryId;
+import org.apache.tajo.ResourceProtos.AllocationResourceProto;
+import org.apache.tajo.ResourceProtos.QueryExecutionRequest;
import org.apache.tajo.TajoProtos;
-import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.WorkerAllocatedResource;
import org.apache.tajo.ipc.QueryMasterProtocol;
import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.QueryExecutionRequestProto;
-import org.apache.tajo.master.rm.WorkerResourceManager;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.rm.TajoResourceManager;
import org.apache.tajo.plan.logical.LogicalRootNode;
-import org.apache.tajo.rpc.*;
+import org.apache.tajo.rpc.CallFuture;
+import org.apache.tajo.rpc.NettyClientBase;
+import org.apache.tajo.rpc.RpcClientManager;
+import org.apache.tajo.rpc.RpcConstants;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.session.Session;
import org.apache.tajo.util.NetUtils;
+import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -53,9 +55,9 @@ public class QueryInProgress {
private LogicalRootNode plan;
- private AtomicBoolean querySubmitted = new AtomicBoolean(false);
+ private volatile boolean querySubmitted;
- private AtomicBoolean stopped = new AtomicBoolean(false);
+ private volatile boolean isStopped;
private QueryInfo queryInfo;
@@ -65,6 +67,8 @@ public class QueryInProgress {
private QueryMasterProtocolService queryMasterRpcClient;
+ private AllocationResourceProto allocationResource;
+
private final Lock readLock;
private final Lock writeLock;
@@ -104,14 +108,16 @@ public class QueryInProgress {
}
public void stopProgress() {
- if(stopped.getAndSet(true)) {
+ if (isStopped) {
return;
+ } else {
+ isStopped = true;
}
LOG.info("=========================================================");
LOG.info("Stop query:" + queryId);
- masterContext.getResourceManager().releaseQueryMaster(queryId);
+ masterContext.getResourceManager().getScheduler().stopQuery(queryId);
RpcClientManager.cleanup(queryMasterRpc);
@@ -122,7 +128,13 @@ public class QueryInProgress {
}
}
- public boolean startQueryMaster() {
+ /**
+ * Connect to QueryMaster and allocate QM resource.
+ *
+ * @param allocation QM resource
+ * @return If there is no available resource, It returns false
+ */
+ protected boolean allocateToQueryMaster(AllocationResourceProto allocation) {
try {
writeLock.lockInterruptibly();
} catch (Exception e) {
@@ -130,19 +142,29 @@ public class QueryInProgress {
return false;
}
try {
- LOG.info("Initializing QueryInProgress for QueryID=" + queryId);
- WorkerResourceManager resourceManager = masterContext.getResourceManager();
- WorkerAllocatedResource resource = resourceManager.allocateQueryMaster(this);
+ TajoResourceManager resourceManager = masterContext.getResourceManager();
+ WorkerConnectionInfo connectionInfo =
+ resourceManager.getRMContext().getNodes().get(allocation.getWorkerId()).getConnectionInfo();
+ try {
+ if(queryMasterRpcClient == null) {
+ connectQueryMaster(connectionInfo);
+ }
+
+ CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>();
+ queryMasterRpcClient.allocateQueryMaster(callFuture.getController(), allocation, callFuture);
- // if no resource to allocate a query master
- if(resource == null) {
- throw new RuntimeException("No Available Resources for QueryMaster");
+ if(!callFuture.get().getValue()) return false;
+
+ } catch (ConnectException ce) {
+ return false;
}
- queryInfo.setQueryMaster(resource.getConnectionInfo().getHost());
- queryInfo.setQueryMasterPort(resource.getConnectionInfo().getQueryMasterPort());
- queryInfo.setQueryMasterclientPort(resource.getConnectionInfo().getClientPort());
- queryInfo.setQueryMasterInfoPort(resource.getConnectionInfo().getHttpInfoPort());
+ LOG.info("Initializing QueryInProgress for QueryID=" + queryId);
+ this.allocationResource = allocation;
+ this.queryInfo.setQueryMaster(connectionInfo.getHost());
+ this.queryInfo.setQueryMasterPort(connectionInfo.getQueryMasterPort());
+ this.queryInfo.setQueryMasterclientPort(connectionInfo.getClientPort());
+ this.queryInfo.setQueryMasterInfoPort(connectionInfo.getHttpInfoPort());
return true;
} catch (Exception e) {
@@ -153,54 +175,58 @@ public class QueryInProgress {
}
}
- private void connectQueryMaster() throws Exception {
- InetSocketAddress addr = NetUtils.createSocketAddr(queryInfo.getQueryMasterHost(), queryInfo.getQueryMasterPort());
- LOG.info("Connect to QueryMaster:" + addr);
-
+ private void connectQueryMaster(WorkerConnectionInfo connectionInfo)
+ throws NoSuchMethodException, ConnectException, ClassNotFoundException {
RpcClientManager.cleanup(queryMasterRpc);
+
+ InetSocketAddress addr = NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getQueryMasterPort());
+ LOG.info("Try to connect to QueryMaster:" + addr);
queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true);
queryMasterRpcClient = queryMasterRpc.getStub();
}
- public void submitQueryToMaster() {
- if(querySubmitted.get()) {
- return;
+ /**
+ * Launch the allocated query to QueryMaster
+ */
+ public boolean submitToQueryMaster() {
+ if(querySubmitted) {
+ return false;
}
try {
writeLock.lockInterruptibly();
} catch (Exception e) {
LOG.error("Failed to lock by exception " + e.getMessage(), e);
- return;
+ return false;
}
try {
- if(queryMasterRpcClient == null) {
- connectQueryMaster();
- }
LOG.info("Call executeQuery to :" +
queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort() + "," + queryId);
- QueryExecutionRequestProto.Builder builder = TajoWorkerProtocol.QueryExecutionRequestProto.newBuilder();
+ QueryExecutionRequest.Builder builder = QueryExecutionRequest.newBuilder();
builder.setQueryId(queryId.getProto())
.setQueryContext(queryInfo.getQueryContext().getProto())
.setSession(session.getProto())
.setExprInJson(PrimitiveProtos.StringProto.newBuilder().setValue(queryInfo.getJsonExpr()))
- .setLogicalPlanJson(PrimitiveProtos.StringProto.newBuilder().setValue(plan.toJson()).build());
+ .setLogicalPlanJson(PrimitiveProtos.StringProto.newBuilder().setValue(plan.toJson()).build())
+ .setAllocation(allocationResource);
CallFuture<PrimitiveProtos.NullProto> callFuture = new CallFuture<PrimitiveProtos.NullProto>();
queryMasterRpcClient.executeQuery(callFuture.getController(), builder.build(), callFuture);
callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
- querySubmitted.set(true);
+ querySubmitted = true;
getQueryInfo().setQueryState(TajoProtos.QueryState.QUERY_MASTER_LAUNCHED);
+ return true;
} catch (Exception e) {
LOG.error("Failed to submit query " + queryId + " to master by exception " + e, e);
catchException(e.getMessage(), e);
} finally {
writeLock.unlock();
}
+ return false;
}
public void catchException(String message, Throwable e) {
@@ -222,10 +248,6 @@ public class QueryInProgress {
}
}
- public boolean isStarted() {
- return !stopped.get() && this.querySubmitted.get();
- }
-
public void heartbeat(QueryInfo queryInfo) {
LOG.info("Received QueryMaster heartbeat:" + queryInfo);
[03/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/ws/rs/responses/WorkerResponse.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/responses/WorkerResponse.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/responses/WorkerResponse.java
index ff0399c..d4c62e4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/responses/WorkerResponse.java
+++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/responses/WorkerResponse.java
@@ -18,10 +18,10 @@
package org.apache.tajo.ws.rs.responses;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
+import org.apache.tajo.master.rm.NodeStatus;
import com.google.gson.annotations.Expose;
+import org.apache.tajo.resource.NodeResource;
public class WorkerResponse {
@@ -35,34 +35,29 @@ public class WorkerResponse {
@Expose private int usedMemoryMB;
@Expose private int usedCpuCoreSlots;
- @Expose private long maxHeap;
- @Expose private long freeHeap;
- @Expose private long totalHeap;
-
@Expose private int numRunningTasks;
@Expose private int numQueryMasterTasks;
@Expose private long lastHeartbeatTime;
- public WorkerResponse(Worker worker) {
- this(worker.getResource());
-
- this.connectionInfo = new WorkerConnectionInfoResponse(worker.getConnectionInfo());
+ public WorkerResponse(NodeStatus nodeStatus) {
+ this(nodeStatus.getTotalResourceCapability(), nodeStatus.getAvailableResource(),
+ nodeStatus.getNumRunningTasks(), nodeStatus.getNumRunningQueryMaster());
+
+ this.connectionInfo = new WorkerConnectionInfoResponse(nodeStatus.getConnectionInfo());
- this.lastHeartbeatTime = worker.getLastHeartbeatTime();
+ this.lastHeartbeatTime = nodeStatus.getLastHeartbeatTime();
}
-
- private WorkerResponse(WorkerResource resource) {
- this.cpuCoreSlots = resource.getCpuCoreSlots();
- this.memoryMB = resource.getMemoryMB();
- this.usedDiskSlots = resource.getUsedDiskSlots();
- this.usedMemoryMB = resource.getUsedMemoryMB();
- this.usedCpuCoreSlots = resource.getUsedCpuCoreSlots();
- this.maxHeap = resource.getMaxHeap();
- this.freeHeap = resource.getFreeHeap();
- this.totalHeap = resource.getTotalHeap();
- this.numRunningTasks = resource.getNumRunningTasks();
- this.numQueryMasterTasks = resource.getNumQueryMasterTasks();
+
+ private WorkerResponse(NodeResource total, NodeResource available, int numRunningTasks, int numQueryMasterTasks) {
+ this.cpuCoreSlots = total.getVirtualCores();
+ this.memoryMB = total.getMemory();
+ this.diskSlots = total.getDisks();
+ this.usedDiskSlots = available.getDisks();
+ this.usedMemoryMB = available.getMemory();
+ this.usedCpuCoreSlots = available.getVirtualCores();
+ this.numRunningTasks = numRunningTasks;
+ this.numQueryMasterTasks = numQueryMasterTasks;
}
public WorkerConnectionInfoResponse getConnectionInfo() {
@@ -89,18 +84,6 @@ public class WorkerResponse {
return usedCpuCoreSlots;
}
- public long getMaxHeap() {
- return maxHeap;
- }
-
- public long getFreeHeap() {
- return freeHeap;
- }
-
- public long getTotalHeap() {
- return totalHeap;
- }
-
public int getNumRunningTasks() {
return numRunningTasks;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/ContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/ContainerProtocol.proto b/tajo-core/src/main/proto/ContainerProtocol.proto
deleted file mode 100644
index df7a450..0000000
--- a/tajo-core/src/main/proto/ContainerProtocol.proto
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *stable* .proto interface.
- */
-
-option java_package = "org.apache.tajo.ipc";
-option java_outer_classname = "ContainerProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-
-package hadoop.yarn;
-
-import "Security.proto";
-import "yarn_protos.proto";
-
-message TajoContainerIdProto {
- optional ApplicationIdProto app_id = 1;
- optional ApplicationAttemptIdProto app_attempt_id = 2;
- optional int32 id = 3;
-}
-
-message TajoContainerProto {
- optional TajoContainerIdProto id = 1;
- optional NodeIdProto nodeId = 2;
- optional string node_http_address = 3;
- optional ResourceProto resource = 4;
- optional PriorityProto priority = 5;
- optional hadoop.common.TokenProto container_token = 6;
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/QueryCoordinatorProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/QueryCoordinatorProtocol.proto b/tajo-core/src/main/proto/QueryCoordinatorProtocol.proto
index 2440e2a..be04dc4 100644
--- a/tajo-core/src/main/proto/QueryCoordinatorProtocol.proto
+++ b/tajo-core/src/main/proto/QueryCoordinatorProtocol.proto
@@ -23,122 +23,13 @@ option java_outer_classname = "QueryCoordinatorProtocol";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
-import "yarn_protos.proto";
-import "tajo_protos.proto";
-import "TajoIdProtos.proto";
-import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
-import "ContainerProtocol.proto";
+import "ResourceProtos.proto";
package hadoop.yarn;
-message ServerStatusProto {
- message System {
- required int32 availableProcessors = 1;
- required int32 freeMemoryMB = 2;
- required int32 maxMemoryMB = 3;
- required int32 totalMemoryMB = 4;
- }
- message Disk {
- required string absolutePath = 1;
- required int64 totalSpace = 2;
- required int64 freeSpace = 3;
- required int64 usableSpace = 4;
- }
-
- message JvmHeap {
- required int64 maxHeap = 1;
- required int64 totalHeap = 2;
- required int64 freeHeap = 3;
- }
-
- required System system = 1;
- required float diskSlots = 2;
- required int32 memoryResourceMB = 3;
- repeated Disk disk = 4;
- required int32 runningTaskNum = 5;
- required JvmHeap jvmHeap = 6;
-}
-
-message TajoHeartbeat {
- required WorkerConnectionInfoProto connectionInfo = 1;
- optional QueryIdProto queryId = 2;
- optional QueryState state = 3;
- optional TableDescProto resultDesc = 4;
- optional string statusMessage = 5;
- optional float queryProgress = 6;
-}
-
-message TajoHeartbeatResponse {
- message ResponseCommand {
- required string command = 1;
- repeated string params = 2;
- }
- required BoolProto heartbeatResult = 1;
- required ClusterResourceSummary clusterResourceSummary = 2;
- optional ResponseCommand responseCommand = 3;
-}
-
-message ClusterResourceSummary {
- required int32 numWorkers = 1;
- required int32 totalDiskSlots = 2;
- required int32 totalCpuCoreSlots = 3;
- required int32 totalMemoryMB = 4;
-
- required int32 totalAvailableDiskSlots = 5;
- required int32 totalAvailableCpuCoreSlots = 6;
- required int32 totalAvailableMemoryMB = 7;
-}
-
-enum ResourceRequestPriority {
- MEMORY = 1;
- DISK = 2;
-}
-
-message WorkerResourceAllocationRequest {
- required QueryIdProto queryId = 1;
- required ResourceRequestPriority resourceRequestPriority = 2;
-
- required int32 numContainers = 3;
-
- required int32 maxMemoryMBPerContainer = 4;
- required int32 minMemoryMBPerContainer = 5;
-
- required float maxDiskSlotPerContainer = 6;
- required float minDiskSlotPerContainer = 7;
-}
-
-message WorkerResourceProto {
- required WorkerConnectionInfoProto connectionInfo = 1;
- required int32 memoryMB = 2 ;
- required float diskSlots = 3;
-}
-
-message WorkerResourcesRequest {
- repeated WorkerResourceProto workerResources = 1;
-}
-
-message WorkerResourceReleaseRequest {
- required ExecutionBlockIdProto executionBlockId = 1;
- repeated TajoContainerIdProto containerIds = 2;
-}
-
-message WorkerAllocatedResource {
- required TajoContainerIdProto containerId = 1;
- required WorkerConnectionInfoProto connectionInfo = 2;
-
- required int32 allocatedMemoryMB = 3;
- required float allocatedDiskSlots = 4;
-}
-
-message WorkerResourceAllocationResponse {
- required QueryIdProto queryId = 1;
- repeated WorkerAllocatedResource workerAllocatedResource = 2;
-}
-
service QueryCoordinatorProtocolService {
- rpc heartbeat(TajoHeartbeat) returns (TajoHeartbeatResponse);
- rpc allocateWorkerResources(WorkerResourceAllocationRequest) returns (WorkerResourceAllocationResponse);
- rpc releaseWorkerResource(WorkerResourceReleaseRequest) returns (BoolProto);
- rpc getAllWorkerResource(NullProto) returns (WorkerResourcesRequest);
+ rpc heartbeat(TajoHeartbeatRequest) returns (TajoHeartbeatResponse);
+ rpc reserveNodeResources(NodeResourceRequest) returns (NodeResourceResponse);
+ rpc getAllWorkers(NullProto) returns (WorkerConnectionsResponse);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/QueryMasterProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/QueryMasterProtocol.proto b/tajo-core/src/main/proto/QueryMasterProtocol.proto
index 855c2c6..f783f06 100644
--- a/tajo-core/src/main/proto/QueryMasterProtocol.proto
+++ b/tajo-core/src/main/proto/QueryMasterProtocol.proto
@@ -21,26 +21,23 @@ option java_outer_classname = "QueryMasterProtocol";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
-import "yarn_protos.proto";
-import "tajo_protos.proto";
import "TajoIdProtos.proto";
-import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
-import "TajoWorkerProtocol.proto";
-import "ContainerProtocol.proto";
+import "ResourceProtos.proto";
package hadoop.yarn;
service QueryMasterProtocolService {
//from Worker
- rpc getTask(GetTaskRequestProto) returns (TaskRequestProto);
rpc statusUpdate (TaskStatusProto) returns (NullProto);
rpc ping (ExecutionBlockIdProto) returns (NullProto);
rpc fatalError(TaskFatalErrorReport) returns (NullProto);
rpc done (TaskCompletionReport) returns (NullProto);
rpc doneExecutionBlock(ExecutionBlockReport) returns (NullProto);
+ rpc getExecutionBlockContext(ExecutionBlockContextRequest) returns (ExecutionBlockContextResponse);
//from TajoMaster's QueryJobManager
rpc killQuery(QueryIdProto) returns (NullProto);
- rpc executeQuery(QueryExecutionRequestProto) returns (NullProto);
+ rpc executeQuery(QueryExecutionRequest) returns (NullProto);
+ rpc allocateQueryMaster(AllocationResourceProto) returns (BoolProto);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/ResourceProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/ResourceProtos.proto b/tajo-core/src/main/proto/ResourceProtos.proto
new file mode 100644
index 0000000..97bf05e
--- /dev/null
+++ b/tajo-core/src/main/proto/ResourceProtos.proto
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.tajo";
+option java_outer_classname = "ResourceProtos";
+option java_generic_services = false;
+option java_generate_equals_and_hash = true;
+
+import "tajo_protos.proto";
+import "TajoIdProtos.proto";
+import "CatalogProtos.proto";
+import "PrimitiveProtos.proto";
+import "Plan.proto";
+
+enum ResponseCommand {
+ NORMAL = 1; //ping
+ MEMBERSHIP = 2; // request membership to worker node
+ ABORT_QUERY = 3; //query master failure
+ SHUTDOWN = 4; // black list
+}
+
+//TODO add node health information
+message NodeStatusProto {
+}
+
+enum ResourceType {
+ LEAF = 1;
+ INTERMEDIATE = 2;
+ QUERYMASTER = 3;
+}
+
+message AllocationResourceProto {
+ required int32 workerId = 1;
+ required NodeResourceProto resource = 2;
+}
+
+
+message ExecutionBlockListProto {
+ repeated ExecutionBlockIdProto executionBlockId = 1;
+}
+
+message TaskAllocationProto {
+ required TaskRequestProto taskRequest = 1;
+ required NodeResourceProto resource = 2;
+}
+
+message TaskRequestProto {
+ required string queryMasterHostAndPort = 1;
+ required TaskAttemptIdProto id = 2;
+ repeated FragmentProto fragments = 3;
+ required string outputTable = 4;
+ required bool clusteredOutput = 5;
+ required LogicalNodeTree plan = 6;
+ optional bool interQuery = 7 [default = false];
+ repeated FetchProto fetches = 8;
+ optional KeyValueSetProto queryContext = 9;
+ optional DataChannelProto dataChannel = 10;
+ optional EnforcerProto enforcer = 11;
+}
+
+message FetchProto {
+ required string host = 1;
+ required int32 port = 2;
+ required ShuffleType type = 3;
+ required ExecutionBlockIdProto executionBlockId = 4;
+ required int32 partitionId = 5;
+ required string name = 6;
+ optional string rangeParams = 7;
+ optional bool hasNext = 8 [default = false];
+
+ //repeated part
+ repeated int32 taskId = 9 [packed=true];
+ repeated int32 attemptId = 10 [packed=true];
+
+ optional int64 offset = 11;
+ optional int64 length = 12;
+}
+
+message TaskStatusProto {
+ required TaskAttemptIdProto id = 1;
+ required string workerName = 2;
+ required float progress = 3;
+ required TaskAttemptState state = 4;
+ optional StatSetProto stats = 5;
+ optional TableStatsProto inputStats = 6;
+ optional TableStatsProto resultStats = 7;
+ repeated ShuffleFileOutput shuffleFileOutputs = 8;
+}
+
+message TaskCompletionReport {
+ required TaskAttemptIdProto id = 1;
+ optional StatSetProto stats = 2;
+ optional TableStatsProto inputStats = 3;
+ optional TableStatsProto resultStats = 4;
+ repeated ShuffleFileOutput shuffleFileOutputs = 5;
+}
+
+message TaskFatalErrorReport {
+ required TaskAttemptIdProto id = 1;
+ optional string errorMessage = 2;
+ optional string errorTrace = 3;
+}
+
+message FailureIntermediateProto {
+ required int64 pagePos = 1;
+ required int32 startRowNum = 2;
+ required int32 endRowNum = 3;
+}
+
+message IntermediateEntryProto {
+ message PageProto {
+ required int64 pos = 1;
+ required int32 length = 2;
+ }
+ required ExecutionBlockIdProto ebId = 1;
+ required int32 taskId = 2;
+ required int32 attemptId = 3;
+ required int32 partId = 4;
+ required string host = 5;
+ required int64 volume = 6;
+ repeated PageProto pages = 7;
+ repeated FailureIntermediateProto failures = 8;
+}
+
+message ExecutionBlockReport {
+ required ExecutionBlockIdProto ebId = 1;
+ required bool reportSuccess = 2;
+ optional string reportErrorMessage = 3;
+ required int32 succeededTasks = 4;
+ repeated IntermediateEntryProto intermediateEntries = 5;
+}
+
+// deprecated
+message TaskResponseProto {
+ required string id = 1;
+ required QueryState status = 2;
+}
+
+message StatusReportProto {
+ required int64 timestamp = 1;
+ required string serverName = 2;
+ repeated TaskStatusProto status = 3;
+ repeated TaskAttemptIdProto pings = 4;
+}
+
+message CommandRequestProto {
+ repeated Command command = 1;
+}
+
+message CommandResponseProto {
+}
+
+message Command {
+ required TaskAttemptIdProto id = 1;
+ required CommandType type = 2;
+}
+
+enum CommandType {
+ PREPARE = 0;
+ LAUNCH = 1;
+ STOP = 2;
+ FINALIZE = 3;
+}
+
+message ShuffleFileOutput {
+ required int32 partId = 1;
+ optional string fileName = 2;
+ optional int64 volume = 3;
+}
+
+message SessionProto {
+ required string session_id = 1;
+ required string username = 2;
+ required string current_database = 3;
+ required int64 last_access_time = 4;
+ required KeyValueSetProto variables = 5;
+}
+
+message NodeHeartbeatRequest {
+ required int32 workerId = 1;
+ optional NodeResourceProto totalResource = 2;
+ optional NodeResourceProto availableResource = 3;
+ optional int32 runningTasks = 4;
+ optional int32 runningQueryMasters = 5;
+ optional WorkerConnectionInfoProto connectionInfo = 6;
+ optional NodeStatusProto status = 7;
+}
+
+message NodeHeartbeatResponse {
+ required ResponseCommand command = 1 [default = NORMAL];
+ optional int32 heartBeatInterval = 2;
+ repeated QueryIdProto queryId = 3;
+}
+
+//deplecated
+message TajoHeartbeatRequest {
+ required WorkerConnectionInfoProto connectionInfo = 1;
+ optional QueryIdProto queryId = 2;
+ optional QueryState state = 3;
+ optional TableDescProto resultDesc = 4;
+ optional string statusMessage = 5;
+ optional float queryProgress = 6;
+}
+
+//deplecated
+message TajoHeartbeatResponse {
+ message ResponseCommand {
+ required string command = 1;
+ repeated string params = 2;
+ }
+ required BoolProto heartbeatResult = 1;
+ optional ResponseCommand responseCommand = 3;
+}
+
+message WorkerConnectionsResponse {
+ repeated WorkerConnectionInfoProto worker = 1;
+}
+
+message NodeResourceRequest {
+ optional string queue = 1;
+ required string userId = 2;
+ required ResourceType type = 3;
+ required int32 priority = 4;
+ required QueryIdProto queryId = 5;
+ required int32 numContainers = 6;
+ required NodeResourceProto capacity = 7;
+ required int32 runningTasks = 8;
+ repeated int32 candidateNodes = 9;
+}
+
+message NodeResourceResponse {
+ required QueryIdProto queryId = 1;
+ repeated AllocationResourceProto resource = 2;
+}
+
+message ExecutionBlockContextRequest {
+ required ExecutionBlockIdProto executionBlockId = 1;
+ required WorkerConnectionInfoProto worker = 2;
+}
+
+message ExecutionBlockContextResponse {
+ required ExecutionBlockIdProto executionBlockId = 1;
+ optional string queryOutputPath = 2;
+
+ required KeyValueSetProto queryContext = 3;
+ required string planJson = 4;
+ required ShuffleType shuffleType = 5;
+}
+
+message StopExecutionBlockRequest {
+ required ExecutionBlockIdProto executionBlockId = 1;
+ optional ExecutionBlockListProto cleanupList = 2;
+}
+
+message BatchAllocationRequest {
+ required ExecutionBlockIdProto executionBlockId = 1;
+ repeated TaskAllocationProto taskRequest = 2;
+}
+
+message BatchAllocationResponse {
+ repeated TaskAllocationProto cancellationTask = 1;
+}
+
+message QueryExecutionRequest {
+ required QueryIdProto queryId = 1;
+ required SessionProto session = 2;
+ required KeyValueSetProto queryContext = 3;
+ required StringProto exprInJson = 4;
+ optional StringProto logicalPlanJson = 5;
+ required AllocationResourceProto allocation = 6;
+}
+
+
+//Task history
+message FetcherHistoryProto {
+ required int64 startTime = 1;
+ optional int64 finishTime = 2;
+ required FetcherState state = 3;
+ required int64 fileLength = 4;
+ required int32 messageReceivedCount = 5;
+}
+
+message TaskHistoryProto {
+ required TaskAttemptIdProto taskAttemptId = 1;
+ required TaskAttemptState state = 2;
+ required float progress = 3;
+ required int64 startTime = 4;
+ required int64 finishTime = 5;
+ required TableStatsProto inputStats = 6;
+ optional TableStatsProto outputStats = 7;
+ optional string outputPath = 8;
+ optional string workingPath = 9;
+ optional int32 finishedFetchCount = 10;
+ optional int32 totalFetchCount = 11;
+ repeated FetcherHistoryProto fetcherHistories = 12;
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/ResourceTrackerProtocol.proto b/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
index dffd8c9..e0ddac2 100644
--- a/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
+++ b/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
@@ -22,45 +22,10 @@ option java_outer_classname = "TajoResourceTrackerProtocol";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
-import "QueryCoordinatorProtocol.proto";
-import "ContainerProtocol.proto";
-import "tajo_protos.proto";
-import "TajoIdProtos.proto";
+import "ResourceProtos.proto";
package hadoop.yarn;
-// deprecated
-message NodeHeartbeat {
- required WorkerConnectionInfoProto connectionInfo = 1;
- optional ServerStatusProto serverStatus = 2;
- optional string statusMessage = 3;
-}
-
-message NodeHeartbeatRequestProto {
- required int32 workerId = 1;
- optional NodeResourceProto totalResource = 2;
- optional NodeResourceProto availableResource = 3;
- optional WorkerConnectionInfoProto connectionInfo = 4;
- optional NodeStatusProto status = 5;
-}
-
-message NodeHeartbeatResponseProto {
- required ResponseCommand command = 1 [default = NORMAL];
- repeated QueryIdProto queryId = 2;
-}
-
-enum ResponseCommand {
- NORMAL = 1; //ping
- MEMBERSHIP = 2; // request membership to worker node
- ABORT_QUERY = 3; //query master failure
- SHUTDOWN = 4; // black list
-}
-
-//TODO add node health information
-message NodeStatusProto {
-}
-
service TajoResourceTrackerProtocolService {
- rpc heartbeat(NodeHeartbeat) returns (TajoHeartbeatResponse);
- rpc nodeHeartbeat(NodeHeartbeatRequestProto) returns (NodeHeartbeatResponseProto);
+ rpc nodeHeartbeat(NodeHeartbeatRequest) returns (NodeHeartbeatResponse);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index 7cc4171..8667702 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -22,357 +22,19 @@ option java_outer_classname = "TajoWorkerProtocol";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
-import "yarn_protos.proto";
-import "tajo_protos.proto";
+
import "TajoIdProtos.proto";
-import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
-import "Plan.proto";
-import "ContainerProtocol.proto";
+import "ResourceProtos.proto";
package hadoop.yarn;
-message SessionProto {
- required string session_id = 1;
- required string username = 2;
- required string current_database = 3;
- required int64 last_access_time = 4;
- required KeyValueSetProto variables = 5;
-}
-
-message TaskStatusProto {
- required TaskAttemptIdProto id = 1;
- required string workerName = 2;
- required float progress = 3;
- required TaskAttemptState state = 4;
- optional StatSetProto stats = 5;
- optional TableStatsProto inputStats = 6;
- optional TableStatsProto resultStats = 7;
- repeated ShuffleFileOutput shuffleFileOutputs = 8;
-}
-
-message TaskCompletionReport {
- required TaskAttemptIdProto id = 1;
- optional StatSetProto stats = 2;
- optional TableStatsProto inputStats = 3;
- optional TableStatsProto resultStats = 4;
- repeated ShuffleFileOutput shuffleFileOutputs = 5;
-}
-
-message TaskFatalErrorReport {
- required TaskAttemptIdProto id = 1;
- optional string errorMessage = 2;
- optional string errorTrace = 3;
-}
-
-message TaskRequestProto {
- required TaskAttemptIdProto id = 1;
- repeated FragmentProto fragments = 2;
- required string outputTable = 3;
- required bool clusteredOutput = 4;
- required LogicalNodeTree plan = 5;
- optional bool interQuery = 6 [default = false];
- repeated FetchProto fetches = 7;
- optional bool shouldDie = 8;
- optional KeyValueSetProto queryContext = 9;
- optional DataChannelProto dataChannel = 10;
- optional EnforcerProto enforcer = 11;
-}
-
-message FetchProto {
- required string host = 1;
- required int32 port = 2;
- required ShuffleType type = 3;
- required ExecutionBlockIdProto executionBlockId = 4;
- required int32 partitionId = 5;
- required string name = 6;
- optional string rangeParams = 7;
- optional bool hasNext = 8 [default = false];
-
- //repeated part
- repeated int32 taskId = 9 [packed=true];
- repeated int32 attemptId = 10 [packed=true];
-
- optional int64 offset = 11;
- optional int64 length = 12;
-}
-
-message FailureIntermediateProto {
- required int64 pagePos = 1;
- required int32 startRowNum = 2;
- required int32 endRowNum = 3;
-}
-
-message IntermediateEntryProto {
- message PageProto {
- required int64 pos = 1;
- required int32 length = 2;
- }
- required ExecutionBlockIdProto ebId = 1;
- required int32 taskId = 2;
- required int32 attemptId = 3;
- required int32 partId = 4;
- required string host = 5;
- required int64 volume = 6;
- repeated PageProto pages = 7;
- repeated FailureIntermediateProto failures = 8;
-}
-
-message ExecutionBlockReport {
- required ExecutionBlockIdProto ebId = 1;
- required bool reportSuccess = 2;
- optional string reportErrorMessage = 3;
- required int32 succeededTasks = 4;
- repeated IntermediateEntryProto intermediateEntries = 5;
-}
-
-// deprecated
-message TaskResponseProto {
- required string id = 1;
- required QueryState status = 2;
-}
-
-message StatusReportProto {
- required int64 timestamp = 1;
- required string serverName = 2;
- repeated TaskStatusProto status = 3;
- repeated TaskAttemptIdProto pings = 4;
-}
-
-message CommandRequestProto {
- repeated Command command = 1;
-}
-
-message CommandResponseProto {
-}
-
-message Command {
- required TaskAttemptIdProto id = 1;
- required CommandType type = 2;
-}
-
-enum CommandType {
- PREPARE = 0;
- LAUNCH = 1;
- STOP = 2;
- FINALIZE = 3;
-}
-
-message ShuffleFileOutput {
- required int32 partId = 1;
- optional string fileName = 2;
- optional int64 volume = 3;
-}
-
-message QueryExecutionRequestProto {
- required QueryIdProto queryId = 1;
- required SessionProto session = 2;
- required KeyValueSetProto queryContext = 3;
- required StringProto exprInJson = 5;
- optional StringProto logicalPlanJson = 6;
-}
-
-// deprecated
-message GetTaskRequestProto {
- required int32 workerId = 1;
- required TajoContainerIdProto containerId = 2;
- required ExecutionBlockIdProto executionBlockId = 3;
-}
-
-message DataChannelProto {
- required ExecutionBlockIdProto srcId = 1;
- required ExecutionBlockIdProto targetId = 2;
-
- required TransmitType transmitType = 3 [default = PULL_TRANSMIT];
- required ShuffleType shuffleType = 4;
-
- optional SchemaProto schema = 5;
-
- repeated ColumnProto shuffleKeys = 7;
- optional int32 numOutputs = 9 [default = 1];
-
- optional string storeType = 10;
-}
-
-message RunExecutionBlockRequestProto {
- required ExecutionBlockIdProto executionBlockId = 1;
- required WorkerConnectionInfoProto queryMaster = 2;
- required string nodeId = 3;
- required string containerId = 4;
- optional string queryOutputPath = 5;
-
- required KeyValueSetProto queryContext = 6;
- required string planJson = 7;
- required ShuffleType shuffleType = 8;
-}
-
-message ExecutionBlockListProto {
- repeated ExecutionBlockIdProto executionBlockId = 1;
-}
-
-message TaskAllocationRequestProto {
- required TaskRequestProto taskRequest = 1;
- required NodeResourceProto resource = 2;
-}
-
-message BatchAllocationRequestProto {
- required ExecutionBlockIdProto executionBlockId = 1;
- repeated TaskAllocationRequestProto taskRequest = 2;
- optional RunExecutionBlockRequestProto executionBlockRequest = 3; //TODO should be refactored
-}
-
-message BatchAllocationResponseProto {
- repeated TaskAllocationRequestProto cancellationTask = 2;
-}
-
service TajoWorkerProtocolService {
rpc ping (TaskAttemptIdProto) returns (BoolProto);
// from QueryMaster(Worker)
- rpc startExecutionBlock(RunExecutionBlockRequestProto) returns (BoolProto);
- rpc stopExecutionBlock(ExecutionBlockIdProto) returns (BoolProto);
+ rpc allocateTasks(BatchAllocationRequest) returns (BatchAllocationResponse);
+ rpc stopExecutionBlock(StopExecutionBlockRequest) returns (BoolProto);
rpc killTaskAttempt(TaskAttemptIdProto) returns (BoolProto);
- rpc cleanup(QueryIdProto) returns (BoolProto);
- rpc cleanupExecutionBlocks(ExecutionBlockListProto) returns (BoolProto);
-}
-
-message EnforceProperty {
- enum EnforceType {
- SORTED_INPUT = 0;
- OUTPUT_DISTINCT = 1;
- GROUP_BY = 2;
- JOIN = 3;
- SORT = 4;
- BROADCAST = 5;
- COLUMN_PARTITION = 6;
- DISTINCT_GROUP_BY = 7;
- }
-
- // Identifies which field is filled in.
- required EnforceType type = 1;
-
- // One of the following will be filled in.
- optional SortedInputEnforce sortedInput = 2;
- optional OutputDistinctEnforce outputDistinct = 3;
- optional GroupbyEnforce groupby = 4;
- optional JoinEnforce join = 5;
- optional SortEnforce sort = 6;
- optional BroadcastEnforce broadcast = 7;
- optional ColumnPartitionEnforcer columnPartition = 8;
- optional DistinctGroupbyEnforcer distinct = 9;
-}
-
-message SortedInputEnforce {
- required string tableName = 1;
- repeated SortSpecProto sortSpecs = 2;
-}
-
-message OutputDistinctEnforce {
-}
-
-message JoinEnforce {
- enum JoinAlgorithm {
- NESTED_LOOP_JOIN = 0;
- BLOCK_NESTED_LOOP_JOIN = 1;
- IN_MEMORY_HASH_JOIN = 2;
- HYBRID_HASH_JOIN = 3;
- MERGE_JOIN = 4;
- }
-
- required int32 nodeId = 1;
- required JoinAlgorithm algorithm = 2;
-}
-
-message GroupbyEnforce {
- enum GroupbyAlgorithm {
- HASH_AGGREGATION = 0;
- SORT_AGGREGATION = 1;
- }
-
- required int32 nodeId = 1;
- required GroupbyAlgorithm algorithm = 2;
- repeated SortSpecProto sortSpecs = 3;
-}
-
-message SortEnforce {
- enum SortAlgorithm {
- IN_MEMORY_SORT = 0;
- MERGE_SORT = 1;
- }
-
- required int32 nodeId = 1;
- required SortAlgorithm algorithm = 2;
-}
-
-message BroadcastEnforce {
- required string tableName = 1;
-}
-
-message ColumnPartitionEnforcer {
- enum ColumnPartitionAlgorithm {
- HASH_PARTITION = 0;
- SORT_PARTITION = 1;
- }
-
- required int32 nodeId = 1;
- required ColumnPartitionAlgorithm algorithm = 2;
-}
-
-message DistinctGroupbyEnforcer {
- enum DistinctAggregationAlgorithm {
- HASH_AGGREGATION = 0;
- SORT_AGGREGATION = 1;
- }
-
- enum MultipleAggregationStage {
- FIRST_STAGE = 0;
- SECOND_STAGE = 1;
- THRID_STAGE = 3;
- }
-
- message SortSpecArray {
- required int32 nodeId = 1;
- repeated SortSpecProto sortSpecs = 2;
- }
- required int32 nodeId = 1;
- required DistinctAggregationAlgorithm algorithm = 2;
- repeated SortSpecArray sortSpecArrays = 3;
- required bool isMultipleAggregation = 4 [default = false];
- optional MultipleAggregationStage multipleAggregationStage = 5;
-}
-
-message EnforcerProto {
- repeated EnforceProperty properties = 1;
-}
-
-message FetcherHistoryProto {
- required int64 startTime = 1;
- optional int64 finishTime = 2;
- required FetcherState state = 3;
- required int64 fileLength = 4;
- required int32 messageReceivedCount = 5;
-}
-
-message TaskHistoryProto {
- required TaskAttemptIdProto taskAttemptId = 1;
- required TaskAttemptState state = 2;
- required float progress = 3;
- required int64 startTime = 4;
- required int64 finishTime = 5;
- required TableStatsProto inputStats = 6;
- optional TableStatsProto outputStats = 7;
- optional string outputPath = 8;
- optional string workingPath = 9;
- optional int32 finishedFetchCount = 10;
- optional int32 totalFetchCount = 11;
- repeated FetcherHistoryProto fetcherHistories = 12;
-}
-
-message TaskRunnerHistoryProto {
- required ExecutionBlockIdProto executionBlockId = 1;
- required string state = 2;
- required string containerId = 3;
- optional int64 startTime = 4;
- optional int64 finishTime = 5;
- repeated TaskHistoryProto taskHistories = 6;
+ rpc stopQuery(QueryIdProto) returns (BoolProto);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/cluster.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/cluster.jsp b/tajo-core/src/main/resources/webapps/admin/cluster.jsp
index 97ca698..7aad8d4 100644
--- a/tajo-core/src/main/resources/webapps/admin/cluster.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/cluster.jsp
@@ -23,14 +23,12 @@
<%@ page import="org.apache.tajo.master.cluster.WorkerConnectionInfo" %>
<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%@ page import="org.apache.tajo.service.TajoMasterInfo" %>
-<%@ page import="org.apache.tajo.master.rm.Worker" %>
-<%@ page import="org.apache.tajo.master.rm.WorkerResource" %>
-<%@ page import="org.apache.tajo.master.rm.WorkerState" %>
+<%@ page import="org.apache.tajo.master.rm.NodeStatus" %>
+<%@ page import="org.apache.tajo.master.rm.NodeState" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.TUtil" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="java.util.*" %>
-<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%@ page import="java.net.InetSocketAddress" %>
<%
@@ -40,37 +38,37 @@
InetSocketAddress socketAddress = new InetSocketAddress(masterName[0], Integer.parseInt(masterName[1]));
String masterLabel = socketAddress.getAddress().getHostName()+ ":" + socketAddress.getPort();
- Map<Integer, Worker> workers = master.getContext().getResourceManager().getWorkers();
- List<Integer> wokerKeys = new ArrayList<Integer>(workers.keySet());
+ Map<Integer, NodeStatus> nodes = master.getContext().getResourceManager().getNodes();
+ List<Integer> wokerKeys = new ArrayList<Integer>(nodes.keySet());
Collections.sort(wokerKeys);
int runningQueryMasterTasks = 0;
- Set<Worker> liveWorkers = new TreeSet<Worker>();
- Set<Worker> deadWorkers = new TreeSet<Worker>();
- Set<Worker> decommissionWorkers = new TreeSet<Worker>();
+ Set<NodeStatus> liveNodes = new TreeSet<NodeStatus>();
+ Set<NodeStatus> deadNodes = new TreeSet<NodeStatus>();
+ Set<NodeStatus> decommissionNodes = new TreeSet<NodeStatus>();
- Set<Worker> liveQueryMasters = new TreeSet<Worker>();
- Set<Worker> deadQueryMasters = new TreeSet<Worker>();
+ Set<NodeStatus> liveQueryMasters = new TreeSet<NodeStatus>();
+ Set<NodeStatus> deadQueryMasters = new TreeSet<NodeStatus>();
- for(Worker eachWorker: workers.values()) {
- liveQueryMasters.add(eachWorker);
- liveWorkers.add(eachWorker);
- runningQueryMasterTasks += eachWorker.getResource().getNumQueryMasterTasks();
+ for(NodeStatus eachNode: nodes.values()) {
+ liveQueryMasters.add(eachNode);
+ liveNodes.add(eachNode);
+ runningQueryMasterTasks += eachNode.getNumRunningQueryMaster();
}
- for (Worker inactiveWorker : master.getContext().getResourceManager().getInactiveWorkers().values()) {
- WorkerState state = inactiveWorker.getState();
+ for (NodeStatus inactiveNode : master.getContext().getResourceManager().getInactiveNodes().values()) {
+ NodeState state = inactiveNode.getState();
- if (state == WorkerState.LOST) {
- deadQueryMasters.add(inactiveWorker);
- deadWorkers.add(inactiveWorker);
- } else if (state == WorkerState.DECOMMISSIONED) {
- decommissionWorkers.add(inactiveWorker);
+ if (state == NodeState.LOST) {
+ deadQueryMasters.add(inactiveNode);
+ deadNodes.add(inactiveNode);
+ } else if (state == NodeState.DECOMMISSIONED) {
+ decommissionNodes.add(inactiveNode);
}
}
- String deadWorkersHtml = deadWorkers.isEmpty() ? "0": "<font color='red'>" + deadWorkers.size() + "</font>";
+ String deadNodesHtml = deadNodes.isEmpty() ? "0": "<font color='red'>" + deadNodes.size() + "</font>";
String deadQueryMastersHtml = deadQueryMasters.isEmpty() ? "0": "<font color='red'>" + deadQueryMasters.size() + "</font>";
ServiceTracker haService = master.getContext().getHAService();
@@ -169,12 +167,11 @@
} else {
%>
<table width="100%" class="border_table" border="1">
- <tr><th>No</th><th>QueryMaster</th><th>Client Port</th><th>Running Query</th><th>Heap(free/total/max)</th><th>Heartbeat</th><th>Status</th></tr>
+ <tr><th>No</th><th>QueryMaster</th><th>Client Port</th><th>Running Query</th><th>Heartbeat</th><th>Status</th></tr>
<%
int no = 1;
- for(Worker queryMaster: liveQueryMasters) {
- WorkerResource resource = queryMaster.getResource();
+ for(NodeStatus queryMaster: liveQueryMasters) {
WorkerConnectionInfo connectionInfo = queryMaster.getConnectionInfo();
String queryMasterHttp = "http://" + connectionInfo.getHost()
+ ":" + connectionInfo.getHttpInfoPort() + "/index.jsp";
@@ -183,8 +180,7 @@
<td width='30' align='right'><%=no++%></td>
<td><a href='<%=queryMasterHttp%>'><%=connectionInfo.getHost() + ":" + connectionInfo.getQueryMasterPort()%></a></td>
<td width='100' align='center'><%=connectionInfo.getClientPort()%></td>
- <td width='200' align='right'><%=resource.getNumQueryMasterTasks()%></td>
- <td width='200' align='center'><%=resource.getFreeHeap()/1024/1024%>/<%=resource.getTotalHeap()/1024/1024%>/<%=resource.getMaxHeap()/1024/1024%> MB</td>
+ <td width='200' align='right'><%=queryMaster.getNumRunningQueryMaster()%></td>
<td width='100' align='right'><%=JSPUtil.getElapsedTime(queryMaster.getLastHeartbeatTime(), System.currentTimeMillis())%></td>
<td width='100' align='center'><%=queryMaster.getState()%></td>
</tr>
@@ -207,7 +203,7 @@
<tr><th>No</th><th>QueryMaster</th>
<%
int no = 1;
- for(Worker queryMaster: deadQueryMasters) {
+ for(NodeStatus queryMaster: deadQueryMasters) {
%>
<tr>
<td width='30' align='right'><%=no++%></td>
@@ -223,34 +219,32 @@
%>
<hr/>
- <h2>Worker</h2>
- <div>Live:<%=liveWorkers.size()%>, Dead: <%=deadWorkersHtml%></div>
+ <h2>Node</h2>
+ <div>Live:<%=liveNodes.size()%>, Dead: <%=deadNodesHtml%></div>
<hr/>
- <h3>Live Workers</h3>
+ <h3>Live Nodes</h3>
<%
- if(liveWorkers.isEmpty()) {
- out.write("No Live Workers\n");
+ if(liveNodes.isEmpty()) {
+ out.write("No Live Nodes\n");
} else {
%>
<table width="100%" class="border_table" border="1">
- <tr><th>No</th><th>Worker</th><th>PullServer<br/>Port</th><th>Running Tasks</th><th>Memory Resource<br/>(used/total)</th><th>Disk Resource<br/>(used/total)</th><th>Heap<br/>(free/total/max)</th><th>Heartbeat</th><th>Status</th></tr>
+ <tr><th>No</th><th>Node</th><th>PullServer<br/>Port</th><th>Running Tasks</th><th>Available</th><th>Total</th><th>Heartbeat</th><th>Status</th></tr>
<%
int no = 1;
- for(Worker worker: liveWorkers) {
- WorkerResource resource = worker.getResource();
- WorkerConnectionInfo connectionInfo = worker.getConnectionInfo();
- String workerHttp = "http://" + connectionInfo.getHost() + ":" + connectionInfo.getHttpInfoPort() + "/index.jsp";
+ for(NodeStatus node: liveNodes) {
+ WorkerConnectionInfo connectionInfo = node.getConnectionInfo();
+ String nodeHttp = "http://" + connectionInfo.getHost() + ":" + connectionInfo.getHttpInfoPort() + "/index.jsp";
%>
<tr>
<td width='30' align='right'><%=no++%></td>
- <td><a href='<%=workerHttp%>'><%=connectionInfo.getHostAndPeerRpcPort()%></a></td>
+ <td><a href='<%=nodeHttp%>'><%=connectionInfo.getHostAndPeerRpcPort()%></a></td>
<td width='80' align='center'><%=connectionInfo.getPullServerPort()%></td>
- <td width='100' align='right'><%=resource.getNumRunningTasks()%></td>
- <td width='150' align='center'><%=resource.getUsedMemoryMB()%>/<%=resource.getMemoryMB()%></td>
- <td width='100' align='center'><%=resource.getUsedDiskSlots()%>/<%=resource.getDiskSlots()%></td>
- <td width='200' align='center'><%=resource.getFreeHeap()/1024/1024%>/<%=resource.getTotalHeap()/1024/1024%>/<%=resource.getMaxHeap()/1024/1024%> MB</td>
- <td width='100' align='right'><%=JSPUtil.getElapsedTime(worker.getLastHeartbeatTime(), System.currentTimeMillis())%></td>
- <td width='100' align='center'><%=worker.getState()%></td>
+ <td width='100' align='right'><%=node.getNumRunningTasks()%></td>
+ <td width='150' align='center'><%=node.getAvailableResource()%></td>
+ <td width='150' align='center'><%=node.getTotalResourceCapability()%></td>
+ <td width='100' align='right'><%=JSPUtil.getElapsedTime(node.getLastHeartbeatTime(), System.currentTimeMillis())%></td>
+ <td width='100' align='center'><%=node.getState()%></td>
</tr>
<%
} //end fo for
@@ -263,25 +257,24 @@
<p/>
<hr/>
<p/>
- <h3>Dead Workers</h3>
+ <h3>Dead Nodes</h3>
<%
- if(deadWorkers.isEmpty()) {
+ if(deadNodes.isEmpty()) {
%>
- No Dead Workers
+ No Dead Nodes
<%
} else {
%>
<table width="300" class="border_table" border="1">
- <tr><th>No</th><th>Worker</th></tr>
+ <tr><th>No</th><th>Node</th></tr>
<%
int no = 1;
- for(Worker worker: deadWorkers) {
- WorkerResource resource = worker.getResource();
+ for(NodeStatus node: deadNodes) {
%>
<tr>
<td width='30' align='right'><%=no++%></td>
- <td><%=worker.getConnectionInfo().getHostAndPeerRpcPort()%></td>
+ <td><%=node.getConnectionInfo().getHostAndPeerRpcPort()%></td>
</tr>
<%
} //end fo for
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/index.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/index.jsp b/tajo-core/src/main/resources/webapps/admin/index.jsp
index bd84283..96facc5 100644
--- a/tajo-core/src/main/resources/webapps/admin/index.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/index.jsp
@@ -21,20 +21,19 @@
<%@ page import="org.apache.hadoop.fs.FileSystem" %>
<%@ page import="org.apache.tajo.conf.TajoConf" %>
-<%@ page import="org.apache.tajo.ipc.QueryCoordinatorProtocol" %>
<%@ page import="org.apache.tajo.master.TajoMaster" %>
-<%@ page import="org.apache.tajo.master.rm.Worker" %>
-<%@ page import="org.apache.tajo.master.rm.WorkerState" %>
<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%@ page import="org.apache.tajo.service.TajoMasterInfo" %>
+<%@ page import="org.apache.tajo.master.rm.NodeStatus" %>
+<%@ page import="org.apache.tajo.master.rm.NodeState" %>
<%@ page import="org.apache.tajo.storage.TablespaceManager" %>
<%@ page import="org.apache.tajo.storage.Tablespace" %>
<%@ page import="org.apache.tajo.util.NetUtils" %>
<%@ page import="org.apache.tajo.util.TUtil" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
+<%@ page import="java.util.List" %>
<%@ page import="java.net.InetSocketAddress" %>
<%@ page import="java.util.Date" %>
-<%@ page import="java.util.List" %>
<%@ page import="java.util.Map" %>
<%
@@ -44,8 +43,8 @@
InetSocketAddress socketAddress = new InetSocketAddress(masterName[0], Integer.parseInt(masterName[1]));
String masterLabel = socketAddress.getAddress().getHostName()+ ":" + socketAddress.getPort();
- Map<Integer, Worker> workers = master.getContext().getResourceManager().getWorkers();
- Map<Integer, Worker> inactiveWorkers = master.getContext().getResourceManager().getInactiveWorkers();
+ Map<Integer, NodeStatus> workers = master.getContext().getResourceManager().getNodes();
+ Map<Integer, NodeStatus> inactiveWorkers = master.getContext().getResourceManager().getInactiveNodes();
int numWorkers = 0;
int numLiveWorkers = 0;
@@ -58,24 +57,21 @@
int runningQueryMasterTask = 0;
- QueryCoordinatorProtocol.ClusterResourceSummary clusterResourceSummary =
- master.getContext().getResourceManager().getClusterResourceSummary();
-
- for(Worker eachWorker: workers.values()) {
+ for(NodeStatus eachWorker: workers.values()) {
numQueryMasters++;
numLiveQueryMasters++;
- runningQueryMasterTask += eachWorker.getResource().getNumQueryMasterTasks();
+ runningQueryMasterTask += eachWorker.getNumRunningQueryMaster();
numWorkers++;
numLiveWorkers++;
}
- for (Worker eachWorker : inactiveWorkers.values()) {
- if (eachWorker.getState() == WorkerState.LOST) {
+ for (NodeStatus eachWorker : inactiveWorkers.values()) {
+ if (eachWorker.getState() == NodeState.LOST) {
numQueryMasters++;
numDeadQueryMasters++;
numWorkers++;
numDeadWorkers++;
- } else if(eachWorker.getState() == WorkerState.DECOMMISSIONED) {
+ } else if(eachWorker.getState() == NodeState.DECOMMISSIONED) {
numDecommissionWorkers++;
}
}
@@ -130,8 +126,8 @@
<tr><td width='150'>System dir:</td><td><%=TajoConf.getSystemDir(master.getContext().getConf())%></td></tr>
<tr><td width='150'>Warehouse dir:</td><td><%=TajoConf.getWarehouseDir(master.getContext().getConf())%></td></tr>
<tr><td width='150'>Staging dir:</td><td><%=TajoConf.getDefaultRootStagingDir(master.getContext().getConf())%></td></tr>
- <tr><td width='150'>Client Service:</td><td><%=NetUtils.normalizeInetSocketAddress(master.getTajoMasterClientService().getBindAddress())%></td></tr>
- <tr><td width='150'>Catalog Service:</td><td><%=master.getCatalogServer().getCatalogServerName()%></td></tr>
+ <tr><td width='150'>Client Service:</td><td><%=master.getTajoMasterClientService().getBindAddress().getHostName() + ":" + master.getTajoMasterClientService().getBindAddress().getPort()%></td></tr>
+ <tr><td width='150'>Catalog Service:</td><td><%=master.getCatalogServer().getBindAddress().getHostName() + ":" + master.getCatalogServer().getBindAddress().getPort()%></td></tr>
<tr><td width='150'>Heap(Free/Total/Max): </td><td><%=Runtime.getRuntime().freeMemory()/1024/1024%> MB / <%=Runtime.getRuntime().totalMemory()/1024/1024%> MB / <%=Runtime.getRuntime().maxMemory()/1024/1024%> MB</td>
<tr><td width='150'>Configuration:</td><td><a href='conf.jsp'>detail...</a></td></tr>
<tr><td width='150'>Environment:</td><td><a href='env.jsp'>detail...</a></td></tr>
@@ -149,7 +145,7 @@
<hr/>
<h3>Cluster Summary</h3>
<table width="100%" class="border_table" border="1">
- <tr><th>Type</th><th>Total</th><th>Live</th><th>Dead</th><th>Running Master</th><th>Memory Resource<br/>(used/total)</th><th>Disk Resource<br/>(used/total)</th></tr>
+ <tr><th>Type</th><th>Total</th><th>Live</th><th>Dead</th><th>Running Master</th><th>Available</th><th>Total</th></tr>
<tr>
<td><a href='cluster.jsp'>Query Master</a></td>
<td align='right'><%=numQueryMasters%></td>
@@ -165,8 +161,8 @@
<td align='right'><%=numLiveWorkers%></td>
<td align='right'><%=numDeadWorkersHtml%></td>
<td align='right'>-</td>
- <td align='center'><%=clusterResourceSummary.getTotalMemoryMB() - clusterResourceSummary.getTotalAvailableMemoryMB()%>/<%=clusterResourceSummary.getTotalMemoryMB()%></td>
- <td align='center'><%=clusterResourceSummary.getTotalDiskSlots() - clusterResourceSummary.getTotalAvailableDiskSlots()%>/<%=clusterResourceSummary.getTotalDiskSlots()%></td>
+ <td align='center'><%=master.getContext().getResourceManager().getScheduler().getClusterResource()%></td>
+ <td align='center'><%=master.getContext().getResourceManager().getScheduler().getMaximumResourceCapability()%></td>
</tr>
<%
if (haService != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/query.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/query.jsp b/tajo-core/src/main/resources/webapps/admin/query.jsp
index ca376bb..0701e34 100644
--- a/tajo-core/src/main/resources/webapps/admin/query.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/query.jsp
@@ -21,7 +21,7 @@
<%@ page import="org.apache.tajo.master.TajoMaster" %>
<%@ page import="org.apache.tajo.master.QueryInProgress" %>
-<%@ page import="org.apache.tajo.master.rm.Worker" %>
+<%@ page import="org.apache.tajo.master.rm.NodeStatus" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.StringUtils" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
@@ -38,11 +38,13 @@
InetSocketAddress socketAddress = new InetSocketAddress(masterName[0], Integer.parseInt(masterName[1]));
String masterLabel = socketAddress.getAddress().getHostName()+ ":" + socketAddress.getPort();
- List<QueryInProgress> runningQueries =
+ List<QueryInProgress> submittedQueries =
new ArrayList<QueryInProgress>(master.getContext().getQueryJobManager().getSubmittedQueries());
+ JSPUtil.sortQueryInProgress(submittedQueries, true);
- runningQueries.addAll(master.getContext().getQueryJobManager().getRunningQueries());
- JSPUtil.sortQueryInProgress(runningQueries, true);
+ List<QueryInProgress> runningQueries =
+ new ArrayList<QueryInProgress>(master.getContext().getQueryJobManager().getRunningQueries());
+ JSPUtil.sortQueryInProgress(runningQueries, true);
int currentPage = 1;
if (request.getParameter("page") != null && !request.getParameter("page").isEmpty()) {
@@ -57,9 +59,8 @@
}
}
- String keyword = request.getParameter("keyword");
- HistoryReader historyReader = master.getContext().getHistoryReader();
- List<QueryInfo> allFinishedQueries = historyReader.getQueries(keyword);
+ List<QueryInfo> allFinishedQueries = new ArrayList<QueryInfo>(master.getContext().getQueryJobManager().getFinishedQueries());
+ Collections.sort(allFinishedQueries, java.util.Collections.reverseOrder());
int numOfFinishedQueries = allFinishedQueries.size();
int totalPage = numOfFinishedQueries % pageSize == 0 ?
@@ -69,15 +70,15 @@
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- Map<Integer, Worker> workers = master.getContext().getResourceManager().getWorkers();
+ Map<Integer, NodeStatus> workers = master.getContext().getResourceManager().getNodes();
Map<String, Integer> portMap = new HashMap<String, Integer>();
Collection<Integer> queryMasters = master.getContext().getResourceManager().getQueryMasters();
if (queryMasters == null || queryMasters.isEmpty()) {
- queryMasters = master.getContext().getResourceManager().getWorkers().keySet();
+ queryMasters = master.getContext().getResourceManager().getNodes().keySet();
}
for(int eachQueryMasterKey: queryMasters) {
- Worker queryMaster = workers.get(eachQueryMasterKey);
+ NodeStatus queryMaster = workers.get(eachQueryMasterKey);
if(queryMaster != null) {
portMap.put(queryMaster.getConnectionInfo().getHost(), queryMaster.getConnectionInfo().getHttpInfoPort());
}
@@ -119,7 +120,39 @@
<%@ include file="header.jsp"%>
<div class='contents'>
<h2>Tajo Master: <%=masterLabel%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
- <hr/>
+ <p />
+ <hr />
+ <h3>Submitted Queries</h3>
+ <%
+ if(submittedQueries.isEmpty()) {
+ out.write("No submitted queries");
+ } else {
+ %>
+ <table width="100%" border="1" class='border_table'>
+ <tr></tr><th>QueryId</th><th>Query Master</th><th>Submitted</th><th>Progress</th><th>Time</th><th>Status</th></th><th>sql</th><th>Kill Query</th></tr>
+ <%
+ for(QueryInProgress eachQuery: submittedQueries) {
+ long time = System.currentTimeMillis() - eachQuery.getQueryInfo().getStartTime();
+ %>
+ <tr>
+ <td><%=eachQuery.getQueryId()%></td>
+ <td><%=eachQuery.getQueryInfo().getQueryMasterHost()%></td>
+ <td><%=df.format(eachQuery.getQueryInfo().getStartTime())%></td>
+ <td><%=(int)(eachQuery.getQueryInfo().getProgress() * 100.0f)%>%</td>
+ <td><%=StringUtils.formatTime(time)%></td>
+ <td><%=eachQuery.getQueryInfo().getQueryState()%></td>
+ <td><%=eachQuery.getQueryInfo().getSql()%></td>
+ <td><input type="submit" value="Kill" onClick="javascript:killQuery('<%=eachQuery.getQueryId()%>');"></td>
+ </tr>
+ <%
+ }
+ %>
+ </table>
+ <%
+ }
+ %>
+ <p/>
+ <hr/>
<h3>Running Queries</h3>
<%
if(runningQueries.isEmpty()) {
@@ -142,7 +175,7 @@
<td><%=StringUtils.formatTime(time)%></td>
<td><%=eachQuery.getQueryInfo().getQueryState()%></td>
<td><%=eachQuery.getQueryInfo().getSql()%></td>
- <td><input id="btnSubmit" type="submit" value="Kill" onClick="javascript:killQuery('<%=eachQuery.getQueryId()%>');"></td>
+ <td><input type="submit" value="Kill" onClick="javascript:killQuery('<%=eachQuery.getQueryId()%>');"></td>
</tr>
<%
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
index 1a58583..49a43d4 100644
--- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
@@ -21,7 +21,6 @@
<%@ page import="org.apache.tajo.master.TajoMaster" %>
<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="javax.xml.ws.Service" %>
<%@ page import="java.net.InetSocketAddress" %>
<%
@@ -335,7 +334,7 @@ function getPage() {
<hr/>
<div id="queryResultTools"></div>
<hr/>
- <div style="display:none;"><form name="dataForm" id="dataForm" method="post" action="getCSV.jsp"><input type="hidden" id="csvData" name="csvData" value="" /></div>
+ <div style="display:none;"><form name="dataForm" id="dataForm" method="post" action="getCSV.jsp"><input type="hidden" id="csvData" name="csvData" value="" /></form></div>
</div>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/querydetail.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/querydetail.jsp b/tajo-core/src/main/resources/webapps/admin/querydetail.jsp
index 099301e..99edca4 100644
--- a/tajo-core/src/main/resources/webapps/admin/querydetail.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/querydetail.jsp
@@ -40,6 +40,8 @@
queryHistory != null ? JSPUtil.sortStageHistories(queryHistory.getStageHistories()) : null;
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String masterLabel = master.getContext().getTajoMasterService().getBindAddress().getHostName()+ ":"
+ + master.getContext().getTajoMasterService().getBindAddress().getPort();
%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
@@ -52,7 +54,7 @@
<body>
<%@ include file="header.jsp"%>
<div class='contents'>
- <h2>Tajo Master: <%=master.getMasterName()%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
+ <h2>Tajo Master: <%= masterLabel%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
<hr/>
<h3><%=queryId%></h3>
<%
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
index d08c0d9..a7b26ae 100644
--- a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
@@ -26,7 +26,7 @@
<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
<%@ page import="org.apache.tajo.util.history.QueryHistory" %>
<%@ page import="org.apache.tajo.util.history.StageHistory" %>
-<%@ page import="org.apache.tajo.master.rm.Worker" %>
+<%@ page import="org.apache.tajo.master.rm.NodeStatus" %>
<%@ page import="java.util.*" %>
<%@ page import="org.apache.tajo.util.history.TaskHistory" %>
<%@ page import="org.apache.tajo.util.*" %>
@@ -74,12 +74,12 @@
status = "ALL";
}
- Collection<Worker> allWorkers = master.getContext().getResourceManager().getWorkers().values();
+ Collection<NodeStatus> allWorkers = master.getContext().getResourceManager().getNodes().values();
- Map<String, Worker> workerMap = new HashMap<String, Worker>();
+ Map<String, NodeStatus> nodeMap = new HashMap<String, NodeStatus>();
if(allWorkers != null) {
- for(Worker eachWorker: allWorkers) {
- workerMap.put(eachWorker.getConnectionInfo().getHostAndPeerRpcPort(), eachWorker);
+ for(NodeStatus eachWorker: allWorkers) {
+ nodeMap.put(eachWorker.getConnectionInfo().getHostAndPeerRpcPort(), eachWorker);
}
}
@@ -99,7 +99,7 @@
totalWriteRows = stage.getTotalWriteRows();
}
- List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId);
+ List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId, NumberUtils.toLong(startTime, 0));
int numTasks = allTasks.size();
int numShuffles = 0;
float totalProgress = 0.0f;
@@ -132,6 +132,9 @@
"&status=" + status + "&sortOrder=" + nextSortOrder + "&sort=";
NumberFormat nf = NumberFormat.getInstance(Locale.US);
+
+ String masterLabel = master.getContext().getTajoMasterService().getBindAddress().getHostName()+ ":"
+ + master.getContext().getTajoMasterService().getBindAddress().getPort();
%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
@@ -144,7 +147,7 @@
<body>
<%@ include file="header.jsp"%>
<div class='contents'>
- <h2>Tajo Master: <%=master.getMasterName()%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
+ <h2>Tajo Master: <%=masterLabel%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
<hr/>
<h3><a href='querydetail.jsp?queryId=<%=queryId%>&startTime=<%=startTime%>'><%=ebId.toString()%></a></h3>
<hr/>
@@ -204,7 +207,7 @@
%>
<div align="right"># Tasks: <%=numOfTasks%> / # Pages: <%=totalPage%></div>
<table border="1" width="100%" class="border_table">
- <tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th><a href='<%=url%>host'>Host</a></th></tr>
+ <tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th>Retry</th><th><a href='<%=url%>host'>Host</a></th></tr>
<%
int rowNo = (currentPage - 1) * pageSize + 1;
for (TaskHistory eachTask: tasks) {
@@ -215,10 +218,10 @@
}
String taskHost = eachTask.getHostAndPort() == null ? "-" : eachTask.getHostAndPort();
if (eachTask.getHostAndPort() != null) {
- Worker worker = workerMap.get(eachTask.getHostAndPort());
- if (worker != null) {
+ NodeStatus nodeStatus = nodeMap.get(eachTask.getHostAndPort());
+ if (nodeStatus != null) {
String[] hostTokens = eachTask.getHostAndPort().split(":");
- taskHost = "<a href='http://" + hostTokens[0] + ":" + worker.getConnectionInfo().getHttpInfoPort() +
+ taskHost = "<a href='http://" + hostTokens[0] + ":" + nodeStatus.getConnectionInfo().getHttpInfoPort() +
"/taskhistory.jsp?taskAttemptId=" + eachTask.getId() + "&startTime=" + eachTask.getLaunchTime() +
"'>" + eachTask.getHostAndPort() + "</a>";
}
@@ -226,12 +229,13 @@
%>
<tr>
- <td><%=rowNo%></td>
+ <td align='center'><%=rowNo%></td>
<td><a href="<%=taskDetailUrl%>"><%=eachTask.getId()%></a></td>
- <td><%=eachTask.getState()%></td>
- <td><%=JSPUtil.percentFormat(eachTask.getProgress())%>%</td>
- <td><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
+ <td align='center'><%=eachTask.getState()%></td>
+ <td align='center'><%=JSPUtil.percentFormat(eachTask.getProgress())%>%</td>
+ <td align='center'><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
<td align='right'><%=eachTask.getLaunchTime() == 0 ? "-" : eachTask.getRunningTime() + " ms"%></td>
+ <td align='center'><%=eachTask.getRetryCount()%></td>
<td><%=taskHost%></td>
</tr>
<%
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/admin/task.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/task.jsp b/tajo-core/src/main/resources/webapps/admin/task.jsp
index 1530572..d3014b1 100644
--- a/tajo-core/src/main/resources/webapps/admin/task.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/task.jsp
@@ -19,13 +19,13 @@
%>
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
-<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
-<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="java.text.SimpleDateFormat" %>
+<%@ page import="org.apache.commons.lang.math.NumberUtils" %>
<%@ page import="org.apache.tajo.master.TajoMaster" %>
+<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
<%@ page import="org.apache.tajo.util.history.TaskHistory" %>
+<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
+<%@ page import="java.text.SimpleDateFormat" %>
<%@ page import="java.util.List" %>
<%
@@ -34,15 +34,16 @@
String queryId = request.getParameter("queryId");
String ebId = request.getParameter("ebid");
-
+ String startTime = request.getParameter("startTime");
String status = request.getParameter("status");
+
if(status == null || status.isEmpty() || "null".equals(status)) {
status = "ALL";
}
String taskAttemptId = request.getParameter("taskAttemptId");
- List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId);
+ List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId, NumberUtils.toLong(startTime, 0));
TaskHistory task = null;
for(TaskHistory eachTask: allTasks) {
@@ -107,12 +108,15 @@
shuffleKey = task.getShuffleKey();
shuffleFileName = task.getShuffleFileName();
}
+
+ String masterLabel = master.getContext().getTajoMasterService().getBindAddress().getHostName()+ ":"
+ + master.getContext().getTajoMasterService().getBindAddress().getPort();
%>
<%@ include file="header.jsp"%>
<div class='contents'>
- <h2>Tajo Master: <%=master.getMasterName()%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
+ <h2>Tajo Master: <%=masterLabel%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
<hr/>
<h3><a href='<%=backUrl%>'><%=ebId%></a></h3>
<hr/>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/index.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/index.jsp b/tajo-core/src/main/resources/webapps/worker/index.jsp
index bc3cb1e..fa00012 100644
--- a/tajo-core/src/main/resources/webapps/worker/index.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/index.jsp
@@ -24,9 +24,7 @@
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="org.apache.tajo.worker.TajoWorker" %>
-<%@ page import="org.apache.tajo.worker.TaskRunner" %>
<%@ page import="java.text.SimpleDateFormat" %>
-<%@ page import="java.util.ArrayList" %>
<%@ page import="java.util.List" %>
<%
@@ -52,6 +50,8 @@
<tr><td width='100'>MaxHeap: </td><td><%=Runtime.getRuntime().maxMemory()/1024/1024%> MB</td>
<tr><td width='100'>TotalHeap: </td><td><%=Runtime.getRuntime().totalMemory()/1024/1024%> MB</td>
<tr><td width='100'>FreeHeap: </td><td><%=Runtime.getRuntime().freeMemory()/1024/1024%> MB</td>
+ <tr><td width='100'>Available Resource: </td><td><%= tajoWorker.getWorkerContext().getNodeResourceManager().getAvailableResource() %></td>
+ <tr><td width='100'>Running Tasks: </td><td><%= tajoWorker.getWorkerContext().getTaskManager().getRunningTasks() %></td>
<tr><td width="100">Configuration:</td><td><a href='conf.jsp'>detail...</a></td></tr>
<tr><td width="100">Environment:</td><td><a href='env.jsp'>detail...</a></td></tr>
<tr><td width="100">Threads:</td><td><a href='thread.jsp'>thread dump...</a></tr>
@@ -62,8 +62,6 @@
List<QueryMasterTask> queryMasterTasks = JSPUtil.sortQueryMasterTask(tajoWorker.getWorkerContext()
.getQueryMasterManagerService().getQueryMaster().getQueryMasterTasks(), true);
- List<QueryMasterTask> finishedQueryMasterTasks = JSPUtil.sortQueryMasterTask(tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getFinishedQueryMasterTasks(), true);
%>
<h3>Running Query</h3>
<%
@@ -90,58 +88,6 @@
} //end of if
%>
</table>
- <p/>
- <hr/>
- <h3>Finished Query</h3>
- <%
- if(finishedQueryMasterTasks.isEmpty()) {
- out.write("No finished query master");
- } else {
- %>
- <table width="100%" border="1" class="border_table">
- <tr><th>QueryId</th><th>Status</th><th>StartTime</th><th>FinishTime</th><th>Progress</th><th>RunTime</th></tr>
- <%
- for(QueryMasterTask eachQueryMasterTask: finishedQueryMasterTasks) {
- Query query = eachQueryMasterTask.getQuery();
- long startTime = query != null ? query.getStartTime() : eachQueryMasterTask.getQuerySubmitTime();
- %>
- <tr>
- <td align='center'><a href='querydetail.jsp?queryId=<%=eachQueryMasterTask.getQueryId()%>'><%=eachQueryMasterTask.getQueryId()%></a></td>
- <td align='center'><%=eachQueryMasterTask.getState()%></td>
- <td align='center'><%=df.format(startTime)%></td>
- <td align='center'><%=(query == null || query.getFinishTime() == 0) ? "-" : df.format(query.getFinishTime())%></td>
- <td align='center'><%=(query == null) ? "-" : (int)(query.getProgress()*100.0f)%>%</td>
- <td align='right'><%=(query == null) ? "-" : JSPUtil.getElapsedTime(query.getStartTime(), query.getFinishTime())%></td>
- </tr>
- <%
- } //end of for
- } //end of if
- %>
- </table>
- <p/>
- <hr/>
-<%
- List<TaskRunner> taskRunners = new ArrayList<TaskRunner>(tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskRunners());
- JSPUtil.sortTaskRunner(taskRunners);
-%>
- <h3>Running Task Containers</h3>
- <a href='taskcontainers.jsp'>[All Task Containers]</a>
- <br/>
- <table width="100%" border="1" class="border_table">
- <tr><th>ContainerId</th><th>StartTime</th><th>FinishTime</th><th>RunTime</th><th>Status</th></tr>
- <%
- for(TaskRunner eachTaskRunner: taskRunners) {
- %>
- <tr>
- <td><a href="tasks.jsp?taskRunnerId=<%=eachTaskRunner.getId()%>"><%=eachTaskRunner.getId()%></a></td>
- <td><%=df.format(eachTaskRunner.getStartTime())%></td>
- <td><%=eachTaskRunner.getFinishTime() == 0 ? "-" : df.format(eachTaskRunner.getFinishTime())%></td>
- <td><%=JSPUtil.getElapsedTime(eachTaskRunner.getStartTime(), eachTaskRunner.getFinishTime())%></td>
- <td><%=eachTaskRunner.getServiceState()%></td>
-<%
- } //end of for
-%>
- </table>
</div>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
index eb40b4f..29862e6 100644
--- a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp
@@ -19,14 +19,12 @@
%>
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-<%@ page import="org.apache.commons.lang.math.NumberUtils" %>
<%@ page import="org.apache.tajo.QueryId" %>
<%@ page import="org.apache.tajo.SessionVars" %>
<%@ page import="org.apache.tajo.querymaster.Query" %>
<%@ page import="org.apache.tajo.querymaster.QueryMasterTask" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
-<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
<%@ page import="org.apache.tajo.util.history.QueryHistory" %>
<%@ page import="org.apache.tajo.util.history.StageHistory" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
@@ -37,29 +35,21 @@
<%
QueryId queryId = TajoIdUtils.parseQueryId(request.getParameter("queryId"));
- String startTime = request.getParameter("startTime");
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
-
- boolean runningQuery = queryMasterTask != null;
+ .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId);
QueryHistory queryHistory = null;
-
- Query query = null;
+ Query query;
if (queryMasterTask != null) {
query = queryMasterTask.getQuery();
if (query != null) {
queryHistory = query.getQueryHistory();
}
} else {
- HistoryReader reader = tajoWorker.getWorkerContext().getHistoryReader();
- queryHistory = reader.getQueryHistory(queryId.toString(), NumberUtils.toLong(startTime, 0));
- }
-
- if (!runningQuery && queryHistory == null) {
- out.write("<script type='text/javascript'>alert('no query history'); history.back(0); </script>");
+ String tajoMasterHttp = request.getScheme() + "://" + JSPUtil.getTajoMasterHttpAddr(tajoWorker.getConfig());
+ response.sendRedirect(tajoMasterHttp + request.getRequestURI() + "?" + request.getQueryString());
return;
}
@@ -67,6 +57,7 @@
queryHistory != null ? JSPUtil.sortStageHistories(queryHistory.getStageHistories()) : null;
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
@@ -82,7 +73,7 @@
<h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
<hr/>
<%
-if (runningQuery && query == null) {
+if (query == null) {
out.write("Query Status: " + queryMasterTask.getState());
String errorMessage = queryMasterTask.getErrorMessage();
if (errorMessage != null && !errorMessage.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/queryplan.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/queryplan.jsp b/tajo-core/src/main/resources/webapps/worker/queryplan.jsp
index 878efe3..422edfc 100644
--- a/tajo-core/src/main/resources/webapps/worker/queryplan.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/queryplan.jsp
@@ -37,7 +37,7 @@
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
+ .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId);
if(queryMasterTask == null) {
out.write("<script type='text/javascript'>alert('no query'); history.back(0); </script>");
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
index 3e3c2c2..2c32006 100644
--- a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
@@ -24,7 +24,6 @@
<%@ page import="org.apache.tajo.TaskAttemptId" %>
<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
<%@ page import="org.apache.tajo.plan.util.PlannerUtil" %>
-<%@ page import="org.apache.tajo.ipc.QueryCoordinatorProtocol" %>
<%@ page import="org.apache.tajo.querymaster.*" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="org.apache.tajo.worker.TajoWorker" %>
@@ -33,6 +32,8 @@
<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
<%@ page import="org.apache.tajo.util.*" %>
<%@ page import="java.util.*" %>
+<%@ page import="org.apache.tajo.TajoProtos" %>
+<%@ page import="org.apache.tajo.master.cluster.WorkerConnectionInfo" %>
<%
String paramQueryId = request.getParameter("queryId");
@@ -60,20 +61,21 @@
}
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
- List<QueryCoordinatorProtocol.WorkerResourceProto> allWorkers = tajoWorker.getWorkerContext()
+ List<TajoProtos.WorkerConnectionInfoProto> allWorkers = tajoWorker.getWorkerContext()
.getQueryMasterManagerService().getQueryMaster().getAllWorker();
- Map<Integer, QueryCoordinatorProtocol.WorkerResourceProto> workerMap = new HashMap<Integer, QueryCoordinatorProtocol.WorkerResourceProto>();
+ Map<Integer, TajoProtos.WorkerConnectionInfoProto> workerMap = new HashMap<Integer, TajoProtos.WorkerConnectionInfoProto>();
if(allWorkers != null) {
- for(QueryCoordinatorProtocol.WorkerResourceProto eachWorker: allWorkers) {
- workerMap.put(eachWorker.getConnectionInfo().getId(), eachWorker);
+ for(TajoProtos.WorkerConnectionInfoProto eachWorker: allWorkers) {
+ workerMap.put(eachWorker.getId(), eachWorker);
}
}
QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
+ .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId);
if(queryMasterTask == null) {
- out.write("<script type='text/javascript'>alert('no query'); history.back(0); </script>");
+ String tajoMasterHttp = request.getScheme() + "://" + JSPUtil.getTajoMasterHttpAddr(tajoWorker.getConfig());
+ response.sendRedirect(tajoMasterHttp + request.getRequestURI() + "?" + request.getQueryString());
return;
}
@@ -220,7 +222,7 @@
%>
<div align="right"># Tasks: <%=numOfTasks%> / # Pages: <%=totalPage%></div>
<table border="1" width="100%" class="border_table">
- <tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th><a href='<%=url%>host'>Host</a></th></tr>
+ <tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th>Retry</th><th><a href='<%=url%>host'>Host</a></th></tr>
<%
for(Task eachTask : tasks) {
int taskSeq = eachTask.getId().getId();
@@ -228,26 +230,22 @@
"&page=" + currentPage + "&pageSize=" + pageSize +
"&taskSeq=" + taskSeq + "&sort=" + sort + "&sortOrder=" + sortOrder;
- String taskHost = eachTask.getSucceededHost() == null ? "-" : eachTask.getSucceededHost();
- if(eachTask.getSucceededHost() != null) {
- QueryCoordinatorProtocol.WorkerResourceProto worker =
- workerMap.get(eachTask.getLastAttempt().getWorkerConnectionInfo().getId());
- if(worker != null) {
- TaskAttempt lastAttempt = eachTask.getLastAttempt();
- if(lastAttempt != null) {
- TaskAttemptId lastAttemptId = lastAttempt.getId();
- taskHost = "<a href='http://" + eachTask.getSucceededHost() + ":" + worker.getConnectionInfo().getHttpInfoPort() + "/taskdetail.jsp?taskAttemptId=" + lastAttemptId + "'>" + eachTask.getSucceededHost() + "</a>";
- }
- }
+ TaskAttempt lastAttempt = eachTask.getLastAttempt();
+ String taskHost = lastAttempt == null ? "-" : lastAttempt.getWorkerConnectionInfo().getHost();
+ if(lastAttempt != null) {
+ WorkerConnectionInfo conn = lastAttempt.getWorkerConnectionInfo();
+ TaskAttemptId lastAttemptId = lastAttempt.getId();
+ taskHost = "<a href='http://" + conn.getHost() + ":" + conn.getHttpInfoPort() + "/taskdetail.jsp?taskAttemptId=" + lastAttemptId + "'>" + conn.getHost() + "</a>";
}
%>
<tr>
- <td><%=rowNo%></td>
+ <td align='center'><%=rowNo%></td>
<td><a href="<%=taskDetailUrl%>"><%=eachTask.getId()%></a></td>
- <td><%=eachTask.getLastAttemptStatus()%></td>
- <td><%=JSPUtil.percentFormat(eachTask.getLastAttempt().getProgress())%>%</td>
- <td><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
+ <td align='center'><%=eachTask.getLastAttemptStatus()%></td>
+ <td align='center'><%=JSPUtil.percentFormat(eachTask.getLastAttempt().getProgress())%>%</td>
+ <td align='center'><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
<td align='right'><%=eachTask.getLaunchTime() == 0 ? "-" : eachTask.getRunningTime() + " ms"%></td>
+ <td align='center'><%=eachTask.getRetryCount()%></td>
<td><%=taskHost%></td>
</tr>
<%
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/task.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/task.jsp b/tajo-core/src/main/resources/webapps/worker/task.jsp
index 17e884a..f2f903b 100644
--- a/tajo-core/src/main/resources/webapps/worker/task.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/task.jsp
@@ -21,16 +21,16 @@
<%@ page import="org.apache.tajo.ExecutionBlockId" %>
<%@ page import="org.apache.tajo.QueryId" %>
+<%@ page import="org.apache.tajo.ResourceProtos.ShuffleFileOutput" %>
<%@ page import="org.apache.tajo.TaskId" %>
<%@ page import="org.apache.tajo.catalog.proto.CatalogProtos" %>
<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
-<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
<%@ page import="org.apache.tajo.querymaster.Query" %>
<%@ page import="org.apache.tajo.querymaster.QueryMasterTask" %>
-<%@ page import="org.apache.tajo.querymaster.Task" %>
<%@ page import="org.apache.tajo.querymaster.Stage" %>
+<%@ page import="org.apache.tajo.querymaster.Task" %>
<%@ page import="org.apache.tajo.storage.DataLocation" %>
-<%@ page import="org.apache.tajo.storage.fragment.FileFragment" %>
+<%@ page import="org.apache.tajo.storage.fragment.Fragment" %>
<%@ page import="org.apache.tajo.storage.fragment.FragmentConvertor" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
@@ -41,7 +41,6 @@
<%@ page import="java.text.SimpleDateFormat" %>
<%@ page import="java.util.Map" %>
<%@ page import="java.util.Set" %>
-<%@ page import="org.apache.tajo.storage.fragment.Fragment" %>
<%
String paramQueryId = request.getParameter("queryId");
@@ -57,10 +56,11 @@
int taskSeq = Integer.parseInt(request.getParameter("taskSeq"));
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
+ .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId);
if(queryMasterTask == null) {
- out.write("<script type='text/javascript'>alert('no query'); history.back(0); </script>");
+ String tajoMasterHttp = request.getScheme() + "://" + JSPUtil.getTajoMasterHttpAddr(tajoWorker.getConfig());
+ response.sendRedirect(tajoMasterHttp + request.getRequestURI() + "?" + request.getQueryString());
return;
}
@@ -131,7 +131,7 @@
String shuffleKey = "-";
String shuffleFileName = "-";
if(numShuffles > 0) {
- TajoWorkerProtocol.ShuffleFileOutput shuffleFileOutputs = task.getShuffleFileOutputs().get(0);
+ ShuffleFileOutput shuffleFileOutputs = task.getShuffleFileOutputs().get(0);
shuffleKey = "" + shuffleFileOutputs.getPartId();
shuffleFileName = shuffleFileOutputs.getFileName();
}
@@ -161,7 +161,7 @@
<tr><td align="right">Launch Time</td><td><%=task.getLaunchTime() == 0 ? "-" : df.format(task.getLaunchTime())%></td></tr>
<tr><td align="right">Finish Time</td><td><%=task.getFinishTime() == 0 ? "-" : df.format(task.getFinishTime())%></td></tr>
<tr><td align="right">Running Time</td><td><%=task.getLaunchTime() == 0 ? "-" : task.getRunningTime() + " ms"%></td></tr>
- <tr><td align="right">Host</td><td><%=task.getSucceededHost() == null ? "-" : task.getSucceededHost()%></td></tr>
+ <tr><td align="right">Host</td><td><%=task.getSucceededWorker() == null ? "-" : task.getSucceededWorker().getHost()%></td></tr>
<tr><td align="right">Shuffles</td><td># Shuffle Outputs: <%=numShuffles%>, Shuffle Key: <%=shuffleKey%>, Shuffle file: <%=shuffleFileName%></td></tr>
<tr><td align="right">Data Locations</td><td><%=dataLocationInfos%></td></tr>
<tr><td align="right">Fragment</td><td><%=fragmentInfo%></td></tr>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/taskcontainers.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/taskcontainers.jsp b/tajo-core/src/main/resources/webapps/worker/taskcontainers.jsp
deleted file mode 100644
index bb5e90d..0000000
--- a/tajo-core/src/main/resources/webapps/worker/taskcontainers.jsp
+++ /dev/null
@@ -1,93 +0,0 @@
-<%
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-%>
-<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
-<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="org.apache.tajo.worker.TajoWorker" %>
-<%@ page import="org.apache.tajo.worker.TaskRunner" %>
-<%@ page import="java.text.SimpleDateFormat" %>
-<%@ page import="java.util.ArrayList" %>
-<%@ page import="java.util.List" %>
-<%@ page import="org.apache.tajo.worker.TaskRunnerHistory" %>
-<%@ page import="org.apache.tajo.worker.TaskRunnerHistory" %>
-
-<%
- TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
-
- List<TaskRunner> taskRunners = new ArrayList<TaskRunner>(tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskRunners());
- List<TaskRunnerHistory> histories = new ArrayList<TaskRunnerHistory>(tajoWorker.getWorkerContext().getTaskRunnerManager().getExecutionBlockHistories());
-
- JSPUtil.sortTaskRunner(taskRunners);
- JSPUtil.sortTaskRunnerHistory(histories);
-
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-%>
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
- <link rel="stylesheet" type = "text/css" href = "/static/style.css" />
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>tajo worker</title>
-</head>
-<body>
-<%@ include file="header.jsp"%>
-<div class='contents'>
- <h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
- <hr/>
- <h3>Running Task Containers</h3>
- <table width="100%" border="1" class="border_table">
- <tr><th>ContainerId</th><th>StartTime</th><th>FinishTime</th><th>RunTime</th><th>Status</th></tr>
-<%
- for(TaskRunner eachTaskRunner: taskRunners) {
-%>
- <tr>
- <td><a href="tasks.jsp?taskRunnerId=<%=eachTaskRunner.getId()%>"><%=eachTaskRunner.getId()%></a></td>
- <td><%=df.format(eachTaskRunner.getStartTime())%></td>
- <td><%=eachTaskRunner.getFinishTime() == 0 ? "-" : df.format(eachTaskRunner.getFinishTime())%></td>
- <td><%=JSPUtil.getElapsedTime(eachTaskRunner.getStartTime(), eachTaskRunner.getFinishTime())%></td>
- <td><%=eachTaskRunner.getServiceState()%></td>
-<%
- }
-%>
- </table>
- <p/>
- <hr/>
- <h3>Finished Task Containers</h3>
- <table width="100%" border="1" class="border_table">
- <tr><th>ContainerId</th><th>StartTime</th><th>FinishTime</th><th>RunTime</th><th>Status</th></tr>
-<%
- for(TaskRunnerHistory history: histories) {
- String taskRunnerId = TaskRunner.getId(history.getExecutionBlockId(), history.getContainerId());
-%>
- <tr>
- <td><a href="tasks.jsp?taskRunnerId=<%=taskRunnerId%>"><%=taskRunnerId%></a></td>
- <td><%=df.format(history.getStartTime())%></td>
- <td><%=history.getFinishTime() == 0 ? "-" : df.format(history.getFinishTime())%></td>
- <td><%=JSPUtil.getElapsedTime(history.getStartTime(), history.getFinishTime())%></td>
- <td><%=history.getState()%></td>
-<%
- }
-%>
- </table>
-</div>
-</body>
-</html>
\ No newline at end of file
[02/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp b/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
index 5c3ce7b..88b7c24 100644
--- a/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
@@ -20,43 +20,30 @@
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
<%@ page import="org.apache.commons.lang.StringUtils" %>
+<%@ page import="org.apache.tajo.ResourceProtos.FetcherHistoryProto" %>
<%@ page import="org.apache.tajo.TaskAttemptId" %>
-<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="org.apache.tajo.worker.*" %>
+<%@ page import="org.apache.tajo.worker.Fetcher" %>
+<%@ page import="org.apache.tajo.worker.TajoWorker" %>
+<%@ page import="org.apache.tajo.worker.Task" %>
+<%@ page import="org.apache.tajo.worker.TaskHistory" %>
<%@ page import="java.text.SimpleDateFormat" %>
<%@ page import="java.util.List" %>
<%
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
- String containerId = request.getParameter("containerId");
String quAttemptId = request.getParameter("taskAttemptId");
TaskAttemptId taskAttemptId = TajoIdUtils.parseTaskAttemptId(quAttemptId);
- Task task = null;
- TaskHistory taskHistory = null;
- if(containerId == null || containerId.isEmpty() || "null".equals(containerId)) {
- task = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskByTaskAttemptId(taskAttemptId);
- if (task != null) {
- taskHistory = task.createTaskHistory();
- } else {
- taskHistory = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskHistoryByTaskAttemptId(taskAttemptId);
- }
+
+ TaskHistory taskHistory;
+ Task task = tajoWorker.getWorkerContext().getTaskManager().getTaskByTaskAttemptId(taskAttemptId);
+ if (task != null) {
+ taskHistory = task.createTaskHistory();
} else {
- TaskRunner runner = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskRunner(containerId);
- if(runner != null) {
- task = runner.getContext().getTask(taskAttemptId);
- if (task != null) {
- taskHistory = task.createTaskHistory();
- } else {
- TaskRunnerHistory history = tajoWorker.getWorkerContext().getTaskRunnerManager().getExcutionBlockHistoryByTaskRunnerId(containerId);
- if(history != null) {
- taskHistory = history.getTaskHistory(taskAttemptId);
- }
- }
- }
+ taskHistory = tajoWorker.getWorkerContext().getTaskManager().getTaskHistory(taskAttemptId.getTaskId());
}
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
%>
@@ -112,7 +99,7 @@
int index = 1;
int pageSize = 1000; //TODO pagination
- List<TajoWorkerProtocol.FetcherHistoryProto> fetcherHistories = taskHistory.getFetcherHistories();
+ List<FetcherHistoryProto> fetcherHistories = taskHistory.getFetcherHistories();
if (fetcherHistories.size() > 0) {
%>
@@ -128,7 +115,7 @@
<th># Messages</th>
</tr>
<%
- for (TajoWorkerProtocol.FetcherHistoryProto eachFetcher : fetcherHistories) {
+ for (FetcherHistoryProto eachFetcher : fetcherHistories) {
%>
<tr>
<td><%=index%>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp b/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
index b7774e8..5dd52d3 100644
--- a/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
@@ -19,7 +19,7 @@
%>
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
+<%@ page import="org.apache.tajo.ResourceProtos.FetcherHistoryProto" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="org.apache.tajo.worker.*" %>
@@ -86,13 +86,13 @@
int index = 1;
int pageSize = 1000; //TODO pagination
- List<TajoWorkerProtocol.FetcherHistoryProto> fetcherHistories = taskHistory.getFetcherHistories();
+ List<FetcherHistoryProto> fetcherHistories = taskHistory.getFetcherHistories();
if (fetcherHistories.size() > 0) {
%>
<table border="1" width="100%" class="border_table">
<tr><th>No</th><th>StartTime</th><th>FinishTime</th><th>RunTime</th><th>Status</th><th>File Length</th><th># Messages</th></tr>
<%
- for (TajoWorkerProtocol.FetcherHistoryProto eachFetcher : fetcherHistories) {
+ for (FetcherHistoryProto eachFetcher : fetcherHistories) {
%>
<tr>
<td><%=index%></td>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/resources/webapps/worker/tasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/tasks.jsp b/tajo-core/src/main/resources/webapps/worker/tasks.jsp
deleted file mode 100644
index ab873cd..0000000
--- a/tajo-core/src/main/resources/webapps/worker/tasks.jsp
+++ /dev/null
@@ -1,107 +0,0 @@
-<%
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-%>
-<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-
-<%@ page import="org.apache.tajo.TaskAttemptId" %>
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
-<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="java.text.SimpleDateFormat" %>
-<%@ page import="java.util.Map" %>
-<%@ page import="org.apache.tajo.worker.*" %>
-
-<%
- String containerId = request.getParameter("taskRunnerId");
- TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
-
- TaskRunner taskRunner = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskRunner(containerId);
- org.apache.tajo.worker.TaskRunnerHistory history = tajoWorker.getWorkerContext().getTaskRunnerManager().getExcutionBlockHistoryByTaskRunnerId(containerId);
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-%>
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
- <link rel="stylesheet" type="text/css" href="/static/style.css"/>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>tajo worker</title>
- <%
- if (taskRunner == null && history == null) {
- %>
- <script type="text/javascript">
- alert("No Task Container for" + containerId);
- document.history.back();
- </script>
-</head>
-</body>
-</html>
-<%
- return;
- }
-%>
-</head>
-<body>
-<%@ include file="header.jsp"%>
-<div class='contents'>
- <h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
- <hr/>
- <h3>Tasks</h3>
- <table width="100%" border="1" class="border_table">
- <tr><th>Id</th><th>StartTime</th><th>FinishTime</th><th>RunTime</th><th>Status</th></tr>
- <%
- if (taskRunner != null) {
- ExecutionBlockContext context = taskRunner.getContext();
-
- for (Map.Entry<TaskAttemptId, Task> entry : context.getTasks().entrySet()) {
- TaskAttemptId taskAttemptId = entry.getKey();
- TaskHistory eachTask = entry.getValue().createTaskHistory();
- %>
- <tr>
- <td>
- <a href="taskdetail.jsp?containerId=<%=containerId%>&taskAttemptId=<%=taskAttemptId%>"><%=taskAttemptId%></a></td>
- <td><%=df.format(eachTask.getStartTime())%></td>
- <td><%=eachTask.getFinishTime() == 0 ? "-" : df.format(eachTask.getFinishTime())%></td>
- <td><%=JSPUtil.getElapsedTime(eachTask.getStartTime(), eachTask.getFinishTime())%></td>
- <td><%=eachTask.getState()%></td>
- </tr>
- <%
- }
- }
-
- if (history != null) {
-
-
- for (Map.Entry<TaskAttemptId, TaskHistory> entry : history.getTaskHistoryMap().entrySet()) {
- TaskAttemptId taskAttemptId = entry.getKey();
- TaskHistory eachTask = entry.getValue();
- %>
- <tr>
- <td><a href="taskdetail.jsp?containerId=<%=containerId%>&taskAttemptId=<%=taskAttemptId%>"><%=taskAttemptId%></a></td>
- <td><%=df.format(eachTask.getStartTime())%></td>
- <td><%=eachTask.getFinishTime() == 0 ? "-" : df.format(eachTask.getFinishTime())%></td>
- <td><%=JSPUtil.getElapsedTime(eachTask.getStartTime(), eachTask.getFinishTime())%></td>
- <td><%=eachTask.getState()%></td>
- </tr>
- <%
- }
- }
- %>
- </table>
-</div>
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index a323f25..995d448 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -231,7 +231,7 @@ public class QueryTestCaseBase {
/* protect a travis stalled build */
System.out.println("Run: " + name.getMethodName() +
" Used memory: " + ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
- / (1024 * 1024)) + "MBytes");
+ / (1024 * 1024)) + " MBytes, Active Threads:" + Thread.activeCount());
}
public QueryTestCaseBase() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 973f1e8..7be0cab 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -19,7 +19,6 @@
package org.apache.tajo;
import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import org.apache.commons.lang.StringUtils;
@@ -40,7 +39,6 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider;
import org.apache.tajo.master.TajoMaster;
-import org.apache.tajo.master.rm.TajoWorkerResourceManager;
import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider;
import org.apache.tajo.querymaster.Query;
import org.apache.tajo.querymaster.QueryMasterTask;
@@ -128,16 +126,10 @@ public class TajoTestingCluster {
conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class);
conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class);
-
- // default resource manager
- if (System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname) != null) {
- String testResourceManager = System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname);
- Preconditions.checkState(testResourceManager.equals(TajoWorkerResourceManager.class.getCanonicalName()));
- conf.set(ConfVars.RESOURCE_MANAGER_CLASS.varname, System.getProperty(ConfVars.RESOURCE_MANAGER_CLASS.varname));
- }
- conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2048);
- conf.setFloat(ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS.varname, 2.0f);
-
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4);
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000);
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3);
+ conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 2);
// Client API RPC
conf.setIntVar(ConfVars.RPC_CLIENT_WORKER_THREAD_NUM, 2);
@@ -145,6 +137,7 @@ public class TajoTestingCluster {
//Client API service RPC Server
conf.setIntVar(ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
conf.setIntVar(ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.REST_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
// Internal RPC Client
conf.setIntVar(ConfVars.INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM, 2);
@@ -157,10 +150,6 @@ public class TajoTestingCluster {
conf.setIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
conf.setIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
- // Resource allocator
- conf.setIntVar(ConfVars.$QUERY_EXECUTE_PARALLEL_MAX, 3);
- conf.setIntVar(ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM, 6); // make twice of parallel_max
-
// Memory cache termination
conf.setIntVar(ConfVars.WORKER_HISTORY_EXPIRE_PERIOD, 1);
@@ -512,7 +501,8 @@ public class TajoTestingCluster {
startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts);
this.dfsCluster.waitClusterUp();
- conf.setInt("hbase.hconnection.threads.core", 50);
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 50);
hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir);
startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, false);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index ace3d0d..d198326 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -50,7 +50,7 @@ import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index dc4dd04..3c9177e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -50,7 +50,7 @@ import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.*;
public class TestFullOuterHashJoinExec {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 8fd61d0..6b32db0 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -52,7 +52,7 @@ import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.*;
public class TestFullOuterMergeJoinExec {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index b9ee06a..f7bc1dd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -51,7 +51,7 @@ import org.junit.Test;
import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.*;
public class TestHashJoinExec {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index c93a1b4..87bcd20 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -50,7 +50,7 @@ import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index c4e7752..d4189bb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -52,7 +52,7 @@ import org.junit.Test;
import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index dff0cbe..ca4b6b7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -69,8 +69,8 @@ import java.util.Set;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.SortEnforce.SortAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.SortEnforce.SortAlgorithm;
import static org.junit.Assert.*;
public class TestPhysicalPlanner {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index d86b229..51f0e76 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -50,7 +50,7 @@ import java.io.IOException;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.JoinEnforce.JoinAlgorithm;
+import static org.apache.tajo.plan.serder.PlanProto.JoinEnforce.JoinAlgorithm;
import static org.junit.Assert.*;
public class TestRightOuterMergeJoinExec {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java
index da0f59d..d769c13 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java
@@ -46,6 +46,7 @@ import static org.junit.Assert.assertTrue;
@Category(IntegrationTest.class)
@RunWith(Parameterized.class)
@NamedTest("TestJoinQuery")
+@net.jcip.annotations.NotThreadSafe
public class TestJoinOnPartitionedTables extends TestJoinQuery {
public TestJoinOnPartitionedTables(String joinOption) throws Exception {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/engine/query/TestOuterJoinQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestOuterJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestOuterJoinQuery.java
index 53dce3c..18eba41 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestOuterJoinQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestOuterJoinQuery.java
@@ -28,8 +28,6 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import java.sql.ResultSet;
-
@Category(IntegrationTest.class)
@RunWith(Parameterized.class)
@NamedTest("TestJoinQuery")
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java
index e8d59d0..207f64d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java
@@ -18,49 +18,9 @@
package org.apache.tajo.master;
-import static org.junit.Assert.*;
-import static org.hamcrest.CoreMatchers.*;
-
-import java.io.File;
-import java.sql.ResultSet;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.tajo.*;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.client.ResultSetUtil;
-import org.apache.tajo.client.TajoClient;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.master.TajoMaster.MasterContext;
-import org.apache.tajo.master.exec.NonForwardQueryResultScanner;
-import org.apache.tajo.master.exec.NonForwardQueryResultSystemScanner;
-import org.apache.tajo.plan.LogicalOptimizer;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.LogicalPlanner;
-import org.apache.tajo.plan.logical.LimitNode;
-import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.storage.RowStoreUtil;
-import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
-import org.apache.tajo.storage.Tuple;
-import org.apache.tajo.util.KeyValueSet;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.hamcrest.TypeSafeDiagnosingMatcher;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.tajo.QueryTestCaseBase;
import org.junit.Test;
-import com.google.protobuf.ByteString;
-
public class TestNonForwardQueryResultSystemScanner extends QueryTestCaseBase {
@Test
public void testGetNextRowsForAggregateFunction() throws Exception {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
index 9910d79..3c378dd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
@@ -20,29 +20,28 @@ package org.apache.tajo.master;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import io.netty.handler.codec.http.QueryStringDecoder;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.QueryId;
+import org.apache.tajo.ResourceProtos.FetchProto;
import org.apache.tajo.TestTajoIds;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.querymaster.Repartitioner;
import org.apache.tajo.querymaster.Task;
import org.apache.tajo.querymaster.Task.IntermediateEntry;
-import org.apache.tajo.querymaster.Repartitioner;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.FetchImpl;
import org.junit.Test;
-import io.netty.handler.codec.http.QueryStringDecoder;
-
import java.net.URI;
import java.util.*;
import static junit.framework.Assert.assertEquals;
-import static org.apache.tajo.querymaster.Repartitioner.FetchGroupMeta;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.HASH_SHUFFLE;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.SCATTERED_HASH_SHUFFLE;
+import static org.apache.tajo.querymaster.Repartitioner.FetchGroupMeta;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
@@ -76,7 +75,7 @@ public class TestRepartitioner {
fetch.setName(sid.toString());
- TajoWorkerProtocol.FetchProto proto = fetch.getProto();
+ FetchProto proto = fetch.getProto();
fetch = new FetchImpl(proto);
assertEquals(proto, fetch.getProto());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
deleted file mode 100644
index 2c997a3..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
+++ /dev/null
@@ -1,454 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import com.google.protobuf.RpcCallback;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.*;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.rpc.NullCallback;
-import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.NodeHeartbeat;
-import static org.junit.Assert.*;
-
-public class TestTajoResourceManager {
- private final PrimitiveProtos.BoolProto BOOL_TRUE = PrimitiveProtos.BoolProto.newBuilder().setValue(true).build();
- private final PrimitiveProtos.BoolProto BOOL_FALSE = PrimitiveProtos.BoolProto.newBuilder().setValue(false).build();
-
- TajoConf tajoConf;
-
- long queryIdTime = System.currentTimeMillis();
- int numWorkers = 5;
- float workerDiskSlots = 5.0f;
- int workerMemoryMB = 512 * 10;
- WorkerResourceAllocationResponse response;
-
- private TajoWorkerResourceManager initResourceManager() throws Exception {
- tajoConf = new org.apache.tajo.conf.TajoConf();
-
- tajoConf.setFloatVar(TajoConf.ConfVars.TAJO_QUERYMASTER_DISK_SLOT, 0.0f);
- tajoConf.setIntVar(TajoConf.ConfVars.TAJO_QUERYMASTER_MEMORY_MB, 512);
- tajoConf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, "localhost:0");
- TajoWorkerResourceManager tajoWorkerResourceManager = new TajoWorkerResourceManager(tajoConf);
- tajoWorkerResourceManager.init(tajoConf);
- tajoWorkerResourceManager.start();
-
- for(int i = 0; i < numWorkers; i++) {
- ServerStatusProto.System system = ServerStatusProto.System.newBuilder()
- .setAvailableProcessors(1)
- .setFreeMemoryMB(workerMemoryMB)
- .setMaxMemoryMB(workerMemoryMB)
- .setTotalMemoryMB(workerMemoryMB)
- .build();
-
- ServerStatusProto.JvmHeap jvmHeap = ServerStatusProto.JvmHeap.newBuilder()
- .setFreeHeap(workerMemoryMB)
- .setMaxHeap(workerMemoryMB)
- .setTotalHeap(workerMemoryMB)
- .build();
-
- ServerStatusProto.Disk disk = ServerStatusProto.Disk.newBuilder()
- .setAbsolutePath("/")
- .setFreeSpace(0)
- .setTotalSpace(0)
- .setUsableSpace(0)
- .build();
-
- List<ServerStatusProto.Disk> disks = new ArrayList<ServerStatusProto.Disk>();
-
- disks.add(disk);
-
- ServerStatusProto serverStatus = ServerStatusProto.newBuilder()
- .setDiskSlots(workerDiskSlots)
- .setMemoryResourceMB(workerMemoryMB)
- .setJvmHeap(jvmHeap)
- .setSystem(system)
- .addAllDisk(disks)
- .setRunningTaskNum(0)
- .build();
-
- WorkerConnectionInfo connectionInfo =
- new WorkerConnectionInfo("host" + (i + 1), 28091, 28092, 21000 + i, 28093, 28080);
- NodeHeartbeat tajoHeartbeat = NodeHeartbeat.newBuilder()
- .setConnectionInfo(connectionInfo.getProto())
- .setServerStatus(serverStatus)
- .build();
-
- tajoWorkerResourceManager.getResourceTracker().heartbeat(null, tajoHeartbeat, NullCallback.get());
- }
-
- return tajoWorkerResourceManager;
- }
-
-
- @Test
- public void testHeartbeat() throws Exception {
- TajoWorkerResourceManager tajoWorkerResourceManager = null;
- try {
- tajoWorkerResourceManager = initResourceManager();
- assertEquals(numWorkers, tajoWorkerResourceManager.getWorkers().size());
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- }
- } finally {
- if (tajoWorkerResourceManager != null) {
- tajoWorkerResourceManager.stop();
- }
- }
- }
-
- @Test
- public void testMemoryResource() throws Exception {
- TajoWorkerResourceManager tajoWorkerResourceManager = null;
- try {
- tajoWorkerResourceManager = initResourceManager();
-
- final int minMemory = 256;
- final int maxMemory = 512;
- float diskSlots = 1.0f;
-
- QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 1);
-
- WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
- .setResourceRequestPriority(ResourceRequestPriority.MEMORY)
- .setNumContainers(60)
- .setQueryId(queryId.getProto())
- .setMaxDiskSlotPerContainer(diskSlots)
- .setMinDiskSlotPerContainer(diskSlots)
- .setMinMemoryMBPerContainer(minMemory)
- .setMaxMemoryMBPerContainer(maxMemory)
- .build();
-
- final CountDownLatch barrier = new CountDownLatch(1);
- final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
- ArrayList<ContainerProtocol.TajoContainerIdProto>();
-
- RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
-
- @Override
- public void run(WorkerResourceAllocationResponse response) {
- TestTajoResourceManager.this.response = response;
- barrier.countDown();
- }
- };
-
- tajoWorkerResourceManager.allocateWorkerResources(request, callBack);
- assertTrue(barrier.await(3, TimeUnit.SECONDS));
-
-
- // assert after callback
- int totalUsedMemory = 0;
- int totalUsedDisks = 0;
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(0, resource.getAvailableMemoryMB());
- assertEquals(0, resource.getAvailableDiskSlots(), 0);
- assertEquals(5.0f, resource.getUsedDiskSlots(), 0);
-
- totalUsedMemory += resource.getUsedMemoryMB();
- totalUsedDisks += resource.getUsedDiskSlots();
- }
-
- assertEquals(workerMemoryMB * numWorkers, totalUsedMemory);
- assertEquals(workerDiskSlots * numWorkers, totalUsedDisks, 0);
-
- assertEquals(numWorkers * 10, response.getWorkerAllocatedResourceList().size());
-
- for(WorkerAllocatedResource eachResource: response.getWorkerAllocatedResourceList()) {
- assertTrue(
- eachResource.getAllocatedMemoryMB() >= minMemory && eachResource.getAllocatedMemoryMB() <= maxMemory);
- containerIds.add(eachResource.getContainerId());
- }
-
- for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
- tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
- }
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
- assertEquals(0, resource.getUsedMemoryMB());
-
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- assertEquals(0.0f, resource.getUsedDiskSlots(), 0);
- }
- } finally {
- if (tajoWorkerResourceManager != null) {
- tajoWorkerResourceManager.stop();
- }
- }
- }
-
- @Test
- public void testMemoryNotCommensurable() throws Exception {
- TajoWorkerResourceManager tajoWorkerResourceManager = null;
-
- try {
- tajoWorkerResourceManager = initResourceManager();
-
- final int minMemory = 200;
- final int maxMemory = 500;
- float diskSlots = 1.0f;
-
- QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 2);
-
- int requiredContainers = 60;
-
- int numAllocatedContainers = 0;
-
- int loopCount = 0;
- while(true) {
- WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
- .setResourceRequestPriority(ResourceRequestPriority.MEMORY)
- .setNumContainers(requiredContainers - numAllocatedContainers)
- .setQueryId(queryId.getProto())
- .setMaxDiskSlotPerContainer(diskSlots)
- .setMinDiskSlotPerContainer(diskSlots)
- .setMinMemoryMBPerContainer(minMemory)
- .setMaxMemoryMBPerContainer(maxMemory)
- .build();
-
- final CountDownLatch barrier = new CountDownLatch(1);
-
- RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
- @Override
- public void run(WorkerResourceAllocationResponse response) {
- TestTajoResourceManager.this.response = response;
- barrier.countDown();
- }
- };
-
- tajoWorkerResourceManager.allocateWorkerResources(request, callBack);
-
- assertTrue(barrier.await(3, TimeUnit.SECONDS));
-
- numAllocatedContainers += TestTajoResourceManager.this.response.getWorkerAllocatedResourceList().size();
-
- //release resource
- for(WorkerAllocatedResource eachResource:
- TestTajoResourceManager.this.response.getWorkerAllocatedResourceList()) {
- assertTrue(
- eachResource.getAllocatedMemoryMB() >= minMemory && eachResource.getAllocatedMemoryMB() <= maxMemory);
- tajoWorkerResourceManager.releaseWorkerResource(eachResource.getContainerId());
- }
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(0, resource.getUsedMemoryMB());
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
-
- assertEquals(0.0f, resource.getUsedDiskSlots(), 0);
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- }
-
- loopCount++;
-
- if(loopCount == 2) {
- assertEquals(requiredContainers, numAllocatedContainers);
- break;
- }
- }
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(0, resource.getUsedMemoryMB());
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
-
- assertEquals(0.0f, resource.getUsedDiskSlots(), 0);
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- }
- } finally {
- if (tajoWorkerResourceManager != null) {
- tajoWorkerResourceManager.stop();
- }
- }
- }
-
- @Test
- public void testDiskResource() throws Exception {
- TajoWorkerResourceManager tajoWorkerResourceManager = null;
-
- try {
- tajoWorkerResourceManager = initResourceManager();
-
- final float minDiskSlots = 1.0f;
- final float maxDiskSlots = 2.0f;
- int memoryMB = 256;
-
- QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 3);
-
- WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
- .setResourceRequestPriority(ResourceRequestPriority.DISK)
- .setNumContainers(60)
- .setQueryId(queryId.getProto())
- .setMaxDiskSlotPerContainer(maxDiskSlots)
- .setMinDiskSlotPerContainer(minDiskSlots)
- .setMinMemoryMBPerContainer(memoryMB)
- .setMaxMemoryMBPerContainer(memoryMB)
- .build();
-
- final CountDownLatch barrier = new CountDownLatch(1);
- final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
- ArrayList<ContainerProtocol.TajoContainerIdProto>();
-
-
- RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
-
- @Override
- public void run(WorkerResourceAllocationResponse response) {
- TestTajoResourceManager.this.response = response;
- barrier.countDown();
- }
- };
-
- tajoWorkerResourceManager.allocateWorkerResources(request, callBack);
- assertTrue(barrier.await(3, TimeUnit.SECONDS));
-
- for(WorkerAllocatedResource eachResource: response.getWorkerAllocatedResourceList()) {
- assertTrue("AllocatedDiskSlot:" + eachResource.getAllocatedDiskSlots(),
- eachResource.getAllocatedDiskSlots() >= minDiskSlots &&
- eachResource.getAllocatedDiskSlots() <= maxDiskSlots);
- containerIds.add(eachResource.getContainerId());
- }
-
- // assert after callback
- int totalUsedDisks = 0;
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- //each worker allocated 3 container (2 disk slot = 2, 1 disk slot = 1)
- assertEquals(0, resource.getAvailableDiskSlots(), 0);
- assertEquals(5.0f, resource.getUsedDiskSlots(), 0);
- assertEquals(256 * 3, resource.getUsedMemoryMB());
-
- totalUsedDisks += resource.getUsedDiskSlots();
- }
-
- assertEquals(workerDiskSlots * numWorkers, totalUsedDisks, 0);
-
- assertEquals(numWorkers * 3, response.getWorkerAllocatedResourceList().size());
-
- for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
- tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
- }
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
- assertEquals(0, resource.getUsedMemoryMB());
-
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- assertEquals(0.0f, resource.getUsedDiskSlots(), 0);
- }
- } finally {
- if (tajoWorkerResourceManager != null) {
- tajoWorkerResourceManager.stop();
- }
- }
- }
-
- @Test
- public void testDiskResourceWithStoppedQuery() throws Exception {
- TajoWorkerResourceManager tajoWorkerResourceManager = null;
-
- try {
- tajoWorkerResourceManager = initResourceManager();
-
- final float minDiskSlots = 1.0f;
- final float maxDiskSlots = 2.0f;
- int memoryMB = 256;
-
- QueryId queryId = QueryIdFactory.newQueryId(queryIdTime, 3);
-
- WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
- .setResourceRequestPriority(ResourceRequestPriority.DISK)
- .setNumContainers(60)
- .setQueryId(queryId.getProto())
- .setMaxDiskSlotPerContainer(maxDiskSlots)
- .setMinDiskSlotPerContainer(minDiskSlots)
- .setMinMemoryMBPerContainer(memoryMB)
- .setMaxMemoryMBPerContainer(memoryMB)
- .build();
-
- final CountDownLatch barrier = new CountDownLatch(1);
- final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
- ArrayList<ContainerProtocol.TajoContainerIdProto>();
-
-
- RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
-
- @Override
- public void run(WorkerResourceAllocationResponse response) {
- TestTajoResourceManager.this.response = response;
- barrier.countDown();
- }
- };
-
- tajoWorkerResourceManager.getRMContext().getStoppedQueryIds().add(queryId);
- tajoWorkerResourceManager.allocateWorkerResources(request, callBack);
- assertFalse(barrier.await(3, TimeUnit.SECONDS));
-
- assertNull(response);
-
- // assert after callback
- int totalUsedDisks = 0;
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- //each worker allocated 3 container (2 disk slot = 2, 1 disk slot = 1)
- assertEquals(5.0f, resource.getAvailableDiskSlots(), 0);
- assertEquals(0, resource.getUsedDiskSlots(), 0);
- assertEquals(0, resource.getUsedMemoryMB());
-
- totalUsedDisks += resource.getUsedDiskSlots();
- }
-
- assertEquals(0, totalUsedDisks, 0);
-
- for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
- tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
- }
-
- for(Worker worker: tajoWorkerResourceManager.getWorkers().values()) {
- WorkerResource resource = worker.getResource();
- assertEquals(workerMemoryMB, resource.getAvailableMemoryMB());
- assertEquals(0, resource.getUsedMemoryMB());
-
- assertEquals(workerDiskSlots, resource.getAvailableDiskSlots(), 0);
- assertEquals(0.0f, resource.getUsedDiskSlots(), 0);
- }
- } finally {
- if (tajoWorkerResourceManager != null) {
- tajoWorkerResourceManager.stop();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java b/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
deleted file mode 100644
index 874461f..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.scheduler;
-
-import org.apache.tajo.QueryId;
-import org.apache.tajo.TajoProtos;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.benchmark.TPCH;
-import org.apache.tajo.client.TajoClient;
-import org.apache.tajo.client.TajoClientUtil;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.ClientProtos;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.sql.ResultSet;
-
-import static org.junit.Assert.*;
-
-public class TestFifoScheduler {
- private static TajoTestingCluster cluster;
- private static TajoConf conf;
- private static TajoClient client;
- private static String query =
- "select l_orderkey, l_partkey from lineitem group by l_orderkey, l_partkey order by l_orderkey";
-
- @BeforeClass
- public static void setUp() throws Exception {
- cluster = new TajoTestingCluster();
- cluster.startMiniClusterInLocal(1);
- conf = cluster.getConfiguration();
- client = cluster.newTajoClient();
- File file = TPCH.getDataFile("lineitem");
- client.executeQueryAndGetResult("create external table default.lineitem (l_orderkey int, l_partkey int) "
- + "using text location 'file://" + file.getAbsolutePath() + "'");
- assertTrue(client.existTable("default.lineitem"));
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- if (client != null) client.close();
- if (cluster != null) cluster.shutdownMiniCluster();
- }
-
- @Test
- public final void testKillScheduledQuery() throws Exception {
- ClientProtos.SubmitQueryResponse res = client.executeQuery(query);
- ClientProtos.SubmitQueryResponse res2 = client.executeQuery(query);
- QueryId queryId = new QueryId(res.getQueryId());
- QueryId queryId2 = new QueryId(res2.getQueryId());
-
- cluster.waitForQuerySubmitted(queryId);
- client.killQuery(queryId2);
- assertEquals(TajoProtos.QueryState.QUERY_KILLED, client.getQueryStatus(queryId2).getState());
- }
-
- @Test
- public final void testForwardedQuery() throws Exception {
- ClientProtos.SubmitQueryResponse res = client.executeQuery(query);
- ClientProtos.SubmitQueryResponse res2 = client.executeQuery("select * from lineitem limit 1");
- assertTrue(res.getIsForwarded());
- assertFalse(res2.getIsForwarded());
-
- QueryId queryId = new QueryId(res.getQueryId());
- QueryId queryId2 = new QueryId(res2.getQueryId());
- cluster.waitForQuerySubmitted(queryId);
-
- assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, client.getQueryStatus(queryId2).getState());
- ResultSet resSet = TajoClientUtil.createResultSet(client, res2, 1);
- assertNotNull(resSet);
- }
-
- @Test
- public final void testScheduledQuery() throws Exception {
- ClientProtos.SubmitQueryResponse res = client.executeQuery("select sleep(1) from lineitem");
- ClientProtos.SubmitQueryResponse res2 = client.executeQuery(query);
- ClientProtos.SubmitQueryResponse res3 = client.executeQuery(query);
- ClientProtos.SubmitQueryResponse res4 = client.executeQuery(query);
-
- QueryId queryId = new QueryId(res.getQueryId());
- QueryId queryId2 = new QueryId(res2.getQueryId());
- QueryId queryId3 = new QueryId(res3.getQueryId());
- QueryId queryId4 = new QueryId(res4.getQueryId());
-
- cluster.waitForQuerySubmitted(queryId);
-
- assertFalse(TajoClientUtil.isQueryComplete(client.getQueryStatus(queryId).getState()));
-
- assertEquals(TajoProtos.QueryState.QUERY_MASTER_INIT, client.getQueryStatus(queryId2).getState());
- assertEquals(TajoProtos.QueryState.QUERY_MASTER_INIT, client.getQueryStatus(queryId3).getState());
- assertEquals(TajoProtos.QueryState.QUERY_MASTER_INIT, client.getQueryStatus(queryId4).getState());
-
- client.killQuery(queryId4);
- client.killQuery(queryId3);
- client.killQuery(queryId2);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java b/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java
new file mode 100644
index 0000000..36fd939
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.master.QueryInfo;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.rm.*;
+import org.apache.tajo.master.scheduler.event.ResourceReserveSchedulerEvent;
+import org.apache.tajo.master.scheduler.event.SchedulerEvent;
+import org.apache.tajo.master.scheduler.event.SchedulerEventType;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.rpc.CallFuture;
+import org.junit.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.*;
+
+import static org.apache.tajo.ResourceProtos.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@NotThreadSafe
+public class TestSimpleScheduler {
+ private CompositeService service;
+ private SimpleScheduler scheduler;
+ private TajoRMContext rmContext;
+ private AsyncDispatcher dispatcher;
+ private TajoConf conf;
+ private int workerNum = 3;
+ private NodeResource nodeResource;
+ private NodeResource totalResource;
+ private Semaphore barrier;
+ private int testDelay = 50;
+ private static ScheduledExecutorService executorService;
+
+ @BeforeClass
+ public static void setupClass() {
+ executorService = Executors.newScheduledThreadPool(10);
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ executorService.shutdown();
+ }
+
+ @Before
+ public void setup() {
+ conf = new TajoConf();
+ nodeResource = NodeResource.createResource(1500, 2, 3);
+ service = new CompositeService(TestSimpleScheduler.class.getSimpleName()) {
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ dispatcher = new AsyncDispatcher();
+ addService(dispatcher);
+
+ rmContext = new TajoRMContext(dispatcher);
+ rmContext.getDispatcher().register(NodeEventType.class,
+ new TajoResourceManager.WorkerEventDispatcher(rmContext));
+
+ barrier = new Semaphore(0);
+ scheduler = new MySimpleScheduler(rmContext, barrier);
+ addService(scheduler);
+ rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);
+
+ for (int i = 0; i < workerNum; i++) {
+ WorkerConnectionInfo conn = new WorkerConnectionInfo("host" + i, 28091 + i, 28092, 21000, 28093, 28080);
+ rmContext.getNodes().putIfAbsent(conn.getId(),
+ new NodeStatus(rmContext, NodeResources.clone(nodeResource), conn));
+ rmContext.getDispatcher().getEventHandler().handle(new NodeEvent(conn.getId(), NodeEventType.STARTED));
+ }
+ super.serviceInit(conf);
+ }
+ };
+ service.init(conf);
+ service.start();
+
+ assertEquals(workerNum, rmContext.getNodes().size());
+ totalResource = NodeResources.createResource(0);
+ for(NodeStatus nodeStatus : rmContext.getNodes().values()) {
+ NodeResources.addTo(totalResource, nodeStatus.getTotalResourceCapability());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ service.stop();
+ }
+
+ @Test
+ public void testInitialCapacity() throws InterruptedException {
+ assertEquals(workerNum, scheduler.getNumClusterNodes());
+ assertEquals(0, scheduler.getRunningQuery());
+
+ assertEquals(totalResource, scheduler.getMaximumResourceCapability());
+ assertEquals(totalResource, scheduler.getClusterResource());
+
+ assertEquals(TajoConf.ConfVars.QUERYMASTER_MINIMUM_MEMORY.defaultIntVal,
+ scheduler.getQMMinimumResourceCapability().getMemory());
+
+ assertEquals(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY.defaultIntVal,
+ scheduler.getMinimumResourceCapability().getMemory());
+ }
+
+ @Test(timeout = 10000)
+ public void testSubmitOneQuery() throws InterruptedException {
+ QuerySchedulingInfo schedulingInfo = new QuerySchedulingInfo("default",
+ "user",
+ QueryIdFactory.newQueryId(System.nanoTime(), 0),
+ 1,
+ System.currentTimeMillis());
+
+ assertEquals(0, scheduler.getRunningQuery());
+
+ scheduler.submitQuery(schedulingInfo);
+ barrier.acquire();
+ assertEquals(1, scheduler.getRunningQuery());
+
+ assertEquals(totalResource, scheduler.getMaximumResourceCapability());
+ assertEquals(totalResource,
+ NodeResources.add(scheduler.getQMMinimumResourceCapability(), scheduler.getClusterResource()));
+ }
+
+ @Test(timeout = 10000)
+ public void testMaximumSubmitQuery() throws InterruptedException {
+ assertEquals(0, scheduler.getRunningQuery());
+ int maximumParallelQuery = scheduler.getResourceCalculator().computeAvailableContainers(
+ scheduler.getMaximumResourceCapability(), scheduler.getQMMinimumResourceCapability());
+
+ int testParallelNum = 10;
+ for (int i = 0; i < testParallelNum; i++) {
+ QuerySchedulingInfo schedulingInfo = new QuerySchedulingInfo("default",
+ "user",
+ QueryIdFactory.newQueryId(System.nanoTime(), 0),
+ 1,
+ System.currentTimeMillis());
+ scheduler.submitQuery(schedulingInfo);
+ }
+
+ barrier.acquire();
+ // allow 50% parallel running
+ assertEquals(Math.floor(maximumParallelQuery * 0.5f), (double) scheduler.getRunningQuery(), 1.0f);
+ assertEquals(testParallelNum, scheduler.getRunningQuery() + scheduler.getQueryQueue().size());
+ }
+
+ @Test(timeout = 10000)
+ public void testReserveResource() throws InterruptedException, ExecutionException {
+ int requestNum = 3;
+ assertEquals(totalResource, scheduler.getMaximumResourceCapability());
+ assertEquals(totalResource, scheduler.getClusterResource());
+
+ QueryId queryId = QueryIdFactory.newQueryId(System.nanoTime(), 0);
+ CallFuture<NodeResourceResponse> callBack = new CallFuture<NodeResourceResponse>();
+ rmContext.getDispatcher().getEventHandler().handle(new ResourceReserveSchedulerEvent(
+ createResourceRequest(queryId, requestNum, new ArrayList<Integer>()), callBack));
+
+ NodeResourceResponse responseProto = callBack.get();
+ assertEquals(queryId, new QueryId(responseProto.getQueryId()));
+ assertEquals(requestNum, responseProto.getResourceCount());
+
+ NodeResource allocations = NodeResources.createResource(0);
+ for (AllocationResourceProto resourceProto : responseProto.getResourceList()) {
+ NodeResources.addTo(allocations, new NodeResource(resourceProto.getResource()));
+ }
+
+ assertEquals(NodeResources.subtract(totalResource, allocations), scheduler.getClusterResource());
+ }
+
+ @Test(timeout = 10000)
+ public void testReserveResourceWithWorkerPriority() throws InterruptedException, ExecutionException {
+ int requestNum = 2;
+ assertEquals(totalResource, scheduler.getMaximumResourceCapability());
+ assertEquals(totalResource, scheduler.getClusterResource());
+
+ List<Integer> targetWorkers = Lists.newArrayList();
+ Map.Entry<Integer, NodeStatus> workerEntry = rmContext.getNodes().entrySet().iterator().next();
+ targetWorkers.add(workerEntry.getKey());
+
+ NodeResource expectResource = NodeResources.multiply(scheduler.getMinimumResourceCapability(), requestNum);
+ assertTrue(NodeResources.fitsIn(expectResource, workerEntry.getValue().getAvailableResource()));
+
+ QueryId queryId = QueryIdFactory.newQueryId(System.nanoTime(), 0);
+ NodeResourceRequest requestProto = createResourceRequest(queryId, requestNum, targetWorkers);
+ CallFuture<NodeResourceResponse> callBack = new CallFuture<NodeResourceResponse>();
+ rmContext.getDispatcher().getEventHandler().handle(new ResourceReserveSchedulerEvent(
+ requestProto, callBack));
+
+ NodeResourceResponse responseProto = callBack.get();
+ assertEquals(queryId, new QueryId(responseProto.getQueryId()));
+ assertEquals(requestNum, responseProto.getResourceCount());
+
+ for (AllocationResourceProto resourceProto : responseProto.getResourceList()) {
+ assertEquals(workerEntry.getKey().intValue(), resourceProto.getWorkerId());
+ }
+ }
+
+ private NodeResourceRequest
+ createResourceRequest(QueryId queryId, int containerNum, List<Integer> candidateWorkers) {
+ NodeResourceRequest.Builder request =
+ NodeResourceRequest.newBuilder();
+ request.setCapacity(scheduler.getMinimumResourceCapability().getProto())
+ .setNumContainers(containerNum)
+ .setPriority(1)
+ .setQueryId(queryId.getProto())
+ .setType(ResourceType.LEAF)
+ .setUserId("test user")
+ .setRunningTasks(0)
+ .addAllCandidateNodes(candidateWorkers)
+ .setQueue("default");
+ return request.build();
+ }
+
+ class MySimpleScheduler extends SimpleScheduler {
+ Semaphore barrier;
+ Map<QueryId, QueryInfo> queryInfoMap = Maps.newHashMap();
+ Map<QueryId, AllocationResourceProto> qmAllocationMap = Maps.newHashMap();
+
+ public MySimpleScheduler(TajoRMContext rmContext, Semaphore barrier) {
+ super(null, rmContext);
+ this.barrier = barrier;
+ }
+
+ @Override
+ public void submitQuery(QuerySchedulingInfo schedulingInfo) {
+ queryInfoMap.put(schedulingInfo.getQueryId(), new QueryInfo(schedulingInfo.getQueryId()) {
+ QueryContext context;
+ @Override
+ public QueryContext getQueryContext() {
+ if(context == null) {
+ context = new QueryContext(conf);
+ context.setUser("user");
+ }
+ return context;
+ }
+ });
+ super.submitQuery(schedulingInfo);
+ }
+
+ @Override
+ protected boolean startQuery(final QueryId queryId, final AllocationResourceProto allocation) {
+ executorService.schedule(new Runnable() {
+ @Override
+ public void run() {
+ barrier.release();
+ qmAllocationMap.put(queryId, allocation);
+ rmContext.getDispatcher().getEventHandler().handle(new SchedulerEvent(SchedulerEventType.RESOURCE_UPDATE));
+ }
+ }, testDelay, TimeUnit.MILLISECONDS);
+ return true;
+ }
+
+ @Override
+ public void handle(SchedulerEvent event) {
+ super.handle(event);
+ barrier.release();
+ }
+
+ @Override
+ protected QueryInfo getQueryInfo(QueryId queryId) {
+ return queryInfoMap.get(queryId);
+ }
+
+ @Override
+ public void stopQuery(QueryId queryId) {
+ queryInfoMap.remove(queryId);
+ AllocationResourceProto allocationResourceProto = qmAllocationMap.remove(queryId);
+ NodeResources.addTo(rmContext.getNodes().get(allocationResourceProto.getWorkerId()).getAvailableResource(),
+ new NodeResource(allocationResourceProto.getResource()));
+ super.stopQuery(queryId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
index 1351716..3e8e230 100644
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
@@ -19,10 +19,11 @@
package org.apache.tajo.querymaster;
import com.google.common.collect.Lists;
-import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.tajo.*;
+import org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.benchmark.TPCH;
import org.apache.tajo.catalog.CatalogService;
@@ -34,26 +35,19 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.engine.query.TaskRequestImpl;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.event.*;
+import org.apache.tajo.master.event.QueryEvent;
+import org.apache.tajo.master.event.QueryEventType;
+import org.apache.tajo.master.event.StageEvent;
+import org.apache.tajo.master.event.StageEventType;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.serder.PlanProto;
-import org.apache.tajo.service.ServiceTracker;
+import org.apache.tajo.resource.NodeResources;
import org.apache.tajo.session.Session;
-import org.apache.tajo.storage.HashShuffleAppenderManager;
import org.apache.tajo.storage.TablespaceManager;
-import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.util.history.HistoryReader;
-import org.apache.tajo.util.history.HistoryWriter;
-import org.apache.tajo.util.metrics.TajoSystemMetrics;
-import org.apache.tajo.worker.ExecutionBlockContext;
-import org.apache.tajo.worker.LegacyTaskImpl;
-import org.apache.tajo.worker.TajoWorker;
-import org.apache.tajo.worker.TaskRunnerManager;
+import org.apache.tajo.worker.*;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -123,7 +117,7 @@ public class TestKillQuery {
QueryMaster qm = cluster.getTajoWorkers().get(0).getWorkerContext().getQueryMaster();
QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(),
- queryId, session, defaultContext, expr.toJson(), dispatch);
+ queryId, session, defaultContext, expr.toJson(), NodeResources.createResource(512), dispatch);
queryMasterTask.init(conf);
queryMasterTask.getQueryTaskContext().getDispatcher().start();
@@ -187,7 +181,7 @@ public class TestKillQuery {
QueryMaster qm = cluster.getTajoWorkers().get(0).getWorkerContext().getQueryMaster();
QueryMasterTask queryMasterTask = new QueryMasterTask(qm.getContext(),
- queryId, session, defaultContext, expr.toJson(), dispatch);
+ queryId, session, defaultContext, expr.toJson(), NodeResources.createResource(512), dispatch);
queryMasterTask.init(conf);
queryMasterTask.getQueryTaskContext().getDispatcher().start();
@@ -223,9 +217,6 @@ public class TestKillQuery {
lastStage.getStateMachine().doTransition(StageEventType.SQ_KILL,
new StageEvent(lastStage.getId(), StageEventType.SQ_KILL));
- lastStage.getStateMachine().doTransition(StageEventType.SQ_CONTAINER_ALLOCATED,
- new StageEvent(lastStage.getId(), StageEventType.SQ_CONTAINER_ALLOCATED));
-
lastStage.getStateMachine().doTransition(StageEventType.SQ_SHUFFLE_REPORT,
new StageEvent(lastStage.getId(), StageEventType.SQ_SHUFFLE_REPORT));
@@ -243,22 +234,21 @@ public class TestKillQuery {
TaskId tid = QueryIdFactory.newTaskId(eid);
final TajoConf conf = new TajoConf();
TaskRequestImpl taskRequest = new TaskRequestImpl();
+ WorkerConnectionInfo queryMaster = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
- taskRequest.set(null, new ArrayList<CatalogProtos.FragmentProto>(),
- null, false, PlanProto.LogicalNodeTree.newBuilder().build(), new QueryContext(conf), null, null);
- taskRequest.setInterQuery();
TaskAttemptId attemptId = new TaskAttemptId(tid, 1);
+ taskRequest.set(attemptId, new ArrayList<CatalogProtos.FragmentProto>(),
+ null, false, PlanProto.LogicalNodeTree.newBuilder().build(), new QueryContext(conf),
+ null, null, queryMaster.getHostAndQMPort());
+ taskRequest.setInterQuery();
- WorkerConnectionInfo queryMaster = new WorkerConnectionInfo("host", 28091, 28092, 21000, 28093, 28080);
- TajoWorkerProtocol.RunExecutionBlockRequestProto.Builder
- requestProto = TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder();
- requestProto.setExecutionBlockId(eid.getProto())
- .setQueryMaster(queryMaster.getProto())
- .setNodeId(queryMaster.getHost()+":" + queryMaster.getQueryMasterPort())
- .setContainerId("test")
- .setQueryContext(new QueryContext(conf).getProto())
+ ExecutionBlockContextResponse.Builder requestProtoBuilder =
+ ExecutionBlockContextResponse.newBuilder();
+ requestProtoBuilder.setExecutionBlockId(eid.getProto())
.setPlanJson("test")
+ .setQueryContext(new QueryContext(conf).getProto())
+ .setQueryOutputPath("testpath")
.setShuffleType(PlanProto.ShuffleType.HASH_SHUFFLE);
TajoWorker.WorkerContext workerContext = new MockWorkerContext() {
@@ -266,13 +256,31 @@ public class TestKillQuery {
public TajoConf getConf() {
return conf;
}
+
+ @Override
+ public TaskManager getTaskManager() {
+ return null;
+ }
+
+ @Override
+ public TaskExecutor getTaskExecuor() {
+ return null;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return null;
+ }
};
- ExecutionBlockContext context =
- new ExecutionBlockContext(workerContext, null, requestProto.build());
+ ExecutionBlockContext context = new MockExecutionBlock(workerContext, requestProtoBuilder.build()) {
+ @Override
+ public Path createBaseDir() throws IOException {
+ return new Path("test");
+ }
+ };
- org.apache.tajo.worker.Task task = new LegacyTaskImpl("test", CommonTestingUtil.getTestDir(), attemptId,
- conf, context, taskRequest);
+ org.apache.tajo.worker.Task task = new TaskImpl(taskRequest, context, null);
task.kill();
assertEquals(TajoProtos.TaskAttemptState.TA_KILLED, task.getTaskContext().getState());
try {
@@ -301,94 +309,4 @@ public class TestKillQuery {
super.dispatch(event);
}
}
-
- abstract class MockWorkerContext implements TajoWorker.WorkerContext {
-
- @Override
- public QueryMaster getQueryMaster() {
- return null;
- }
-
- public abstract TajoConf getConf();
-
- @Override
- public ServiceTracker getServiceTracker() {
- return null;
- }
-
- @Override
- public QueryMasterManagerService getQueryMasterManagerService() {
- return null;
- }
-
- @Override
- public TaskRunnerManager getTaskRunnerManager() {
- return null;
- }
-
- @Override
- public CatalogService getCatalog() {
- return null;
- }
-
- @Override
- public WorkerConnectionInfo getConnectionInfo() {
- return null;
- }
-
- @Override
- public String getWorkerName() {
- return null;
- }
-
- @Override
- public LocalDirAllocator getLocalDirAllocator() {
- return null;
- }
-
- @Override
- public QueryCoordinatorProtocol.ClusterResourceSummary getClusterResource() {
- return null;
- }
-
- @Override
- public TajoSystemMetrics getWorkerSystemMetrics() {
- return null;
- }
-
- @Override
- public HashShuffleAppenderManager getHashShuffleAppenderManager() {
- return null;
- }
-
- @Override
- public HistoryWriter getTaskHistoryWriter() {
- return null;
- }
-
- @Override
- public HistoryReader getHistoryReader() {
- return null;
- }
-
- @Override
- public void cleanup(String strPath) {
-
- }
-
- @Override
- public void cleanupTemporalDirectories() {
-
- }
-
- @Override
- public void setClusterResource(QueryCoordinatorProtocol.ClusterResourceSummary clusterResource) {
-
- }
-
- @Override
- public void setNumClusterNodes(int numClusterNodes) {
-
- }
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/querymaster/TestQueryState.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestQueryState.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestQueryState.java
index a822e42..d2241b4 100644
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestQueryState.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestQueryState.java
@@ -18,28 +18,36 @@
package org.apache.tajo.querymaster;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.tajo.*;
+import org.apache.tajo.client.QueryStatus;
import org.apache.tajo.client.TajoClient;
+import org.apache.tajo.client.TajoClientUtil;
import org.apache.tajo.ipc.ClientProtos;
-import org.apache.tajo.master.QueryManager;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.*;
@Category(IntegrationTest.class)
+@NotThreadSafe
public class TestQueryState {
private static TajoTestingCluster cluster;
private static TajoClient client;
@BeforeClass
- public static void setUp() throws Exception {
+ public static void setUpClass() throws Exception {
cluster = TpchTestBase.getInstance().getTestingCluster();
client = cluster.newTajoClient();
}
+ @AfterClass
+ public static void tearDownClass() {
+ client.close();
+ }
+
@Test(timeout = 10000)
public void testSucceededState() throws Exception {
String queryStr = "select l_orderkey from lineitem group by l_orderkey order by l_orderkey";
@@ -61,16 +69,21 @@ public class TestQueryState {
ClientProtos.SubmitQueryResponse res = client.executeQuery(queryStr);
QueryId queryId = new QueryId(res.getQueryId());
- cluster.waitForQuerySubmitted(queryId);
+
+ QueryStatus queryState = client.getQueryStatus(queryId);
+ while (!TajoClientUtil.isQueryComplete(queryState.getState())) {
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ fail("Query state : " + queryState);
+ }
+ queryState = client.getQueryStatus(queryId);
+ }
QueryMasterTask qmt = cluster.getQueryMasterTask(queryId);
Query query = qmt.getQuery();
- // wait for query complete
- cluster.waitForQueryState(query, TajoProtos.QueryState.QUERY_SUCCEEDED, 100);
-
assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, qmt.getState());
-
assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, query.getSynchronizedState());
assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, query.getState());
@@ -80,13 +93,6 @@ public class TestQueryState {
assertEquals(StageState.SUCCEEDED, stage.getState());
}
- /* wait for heartbeat from QueryMaster */
- QueryManager queryManager = cluster.getMaster().getContext().getQueryJobManager();
- for (; ; ) {
- if (queryManager.getFinishedQuery(queryId) != null) break;
- else Thread.sleep(100);
- }
-
/* get status from TajoMaster */
assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, client.getQueryStatus(queryId).getState());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java b/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java
index 51622b5..0ce2646 100644
--- a/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java
+++ b/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java
@@ -124,6 +124,7 @@ public class TestSystemMetrics {
}
assertEquals(2, lines.size());
+ tajoSystemMetrics.stop();
}
@After
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockExecutionBlock.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockExecutionBlock.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockExecutionBlock.java
index 9d4e1f3..7d7fb1a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockExecutionBlock.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockExecutionBlock.java
@@ -18,16 +18,16 @@
package org.apache.tajo.worker;
+import org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import java.io.IOException;
public class MockExecutionBlock extends ExecutionBlockContext {
public MockExecutionBlock(TajoWorker.WorkerContext workerContext,
- TajoWorkerProtocol.RunExecutionBlockRequestProto request) throws IOException {
- super(workerContext, null, request);
+ ExecutionBlockContextResponse request) throws IOException {
+ super(workerContext, request, null);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java
index 18b9405..8c8427d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java
@@ -19,11 +19,11 @@
package org.apache.tajo.worker;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.ResourceProtos.TaskAllocationProto;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.resource.NodeResources;
@@ -38,8 +38,8 @@ public class MockNodeResourceManager extends NodeResourceManager {
volatile boolean enableTaskHandlerEvent = true;
private final Semaphore barrier;
- public MockNodeResourceManager(Semaphore barrier, Dispatcher dispatcher, EventHandler taskEventHandler) {
- super(dispatcher, taskEventHandler);
+ public MockNodeResourceManager(Semaphore barrier, Dispatcher dispatcher, TajoWorker.WorkerContext workerContext) {
+ super(dispatcher, workerContext);
this.barrier = barrier;
}
@@ -50,14 +50,7 @@ public class MockNodeResourceManager extends NodeResourceManager {
}
@Override
- protected void startExecutionBlock(TajoWorkerProtocol.RunExecutionBlockRequestProto request) {
- if(enableTaskHandlerEvent) {
- super.startExecutionBlock(request);
- }
- }
-
- @Override
- protected void startTask(TajoWorkerProtocol.TaskRequestProto request, NodeResource resource) {
+ protected void startTask(TaskRequestProto request, NodeResource resource) {
if(enableTaskHandlerEvent) {
super.startTask(request, resource);
}
@@ -70,24 +63,23 @@ public class MockNodeResourceManager extends NodeResourceManager {
enableTaskHandlerEvent = flag;
}
- protected static Queue<TajoWorkerProtocol.TaskAllocationRequestProto> createTaskRequests(
+ protected static Queue<TaskAllocationProto> createTaskRequests(
ExecutionBlockId ebId, int memory, int size) {
- Queue<TajoWorkerProtocol.TaskAllocationRequestProto>
- requestProtoList = new LinkedBlockingQueue<TajoWorkerProtocol.TaskAllocationRequestProto>();
+ Queue<TaskAllocationProto>
+ requestProtoList = new LinkedBlockingQueue<TaskAllocationProto>();
for (int i = 0; i < size; i++) {
TaskAttemptId taskAttemptId = QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(ebId, i), 0);
- TajoWorkerProtocol.TaskRequestProto.Builder builder =
- TajoWorkerProtocol.TaskRequestProto.newBuilder();
+ TaskRequestProto.Builder builder = TaskRequestProto.newBuilder();
+ builder.setQueryMasterHostAndPort("localhost:0");
builder.setId(taskAttemptId.getProto());
- builder.setShouldDie(true);
builder.setOutputTable("");
builder.setPlan(PlanProto.LogicalNodeTree.newBuilder());
builder.setClusteredOutput(false);
- requestProtoList.add(TajoWorkerProtocol.TaskAllocationRequestProto.newBuilder()
+ requestProtoList.add(TaskAllocationProto.newBuilder()
.setResource(NodeResources.createResource(memory).getProto())
.setTaskRequest(builder.build()).build());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeStatusUpdater.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeStatusUpdater.java
index dfcfd4f..634398f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeStatusUpdater.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockNodeStatusUpdater.java
@@ -21,8 +21,7 @@ package org.apache.tajo.worker;
import com.google.common.collect.Maps;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.ipc.TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService;
import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.resource.NodeResources;
@@ -30,7 +29,7 @@ import java.net.ConnectException;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.*;
+import static org.apache.tajo.ResourceProtos.*;
public class MockNodeStatusUpdater extends NodeStatusUpdater {
@@ -39,9 +38,8 @@ public class MockNodeStatusUpdater extends NodeStatusUpdater {
private Map<Integer, NodeResource> resources = Maps.newHashMap();
private MockResourceTracker resourceTracker;
- public MockNodeStatusUpdater(CountDownLatch barrier, TajoWorker.WorkerContext workerContext,
- NodeResourceManager resourceManager) {
- super(workerContext, resourceManager);
+ public MockNodeStatusUpdater(CountDownLatch barrier, TajoWorker.WorkerContext workerContext) {
+ super(workerContext);
this.barrier = barrier;
this.resourceTracker = new MockResourceTracker();
}
@@ -58,7 +56,7 @@ public class MockNodeStatusUpdater extends NodeStatusUpdater {
}
class MockResourceTracker implements TajoResourceTrackerProtocolService.Interface {
- private NodeHeartbeatRequestProto lastRequest;
+ private NodeHeartbeatRequest lastRequest;
protected Map<Integer, NodeResource> getTotalResource() {
return membership;
@@ -68,21 +66,15 @@ public class MockNodeStatusUpdater extends NodeStatusUpdater {
return membership;
}
- protected NodeHeartbeatRequestProto getLastRequest() {
+ protected NodeHeartbeatRequest getLastRequest() {
return lastRequest;
}
@Override
- public void heartbeat(RpcController controller, NodeHeartbeat request,
- RpcCallback<QueryCoordinatorProtocol.TajoHeartbeatResponse> done) {
+ public void nodeHeartbeat(RpcController controller, NodeHeartbeatRequest request,
+ RpcCallback<NodeHeartbeatResponse> done) {
- }
-
- @Override
- public void nodeHeartbeat(RpcController controller, NodeHeartbeatRequestProto request,
- RpcCallback<NodeHeartbeatResponseProto> done) {
-
- NodeHeartbeatResponseProto.Builder response = NodeHeartbeatResponseProto.newBuilder();
+ NodeHeartbeatResponse.Builder response = NodeHeartbeatResponse.newBuilder();
if (membership.containsKey(request.getWorkerId())) {
if (request.hasAvailableResource()) {
NodeResource resource = resources.get(request.getWorkerId());
[10/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java
index 0c8d8ce..b4ed5fd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java
@@ -18,7 +18,6 @@
package org.apache.tajo.master;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.map.LRUMap;
@@ -30,12 +29,15 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.ResourceProtos;
+import org.apache.tajo.ResourceProtos.AllocationResourceProto;
+import org.apache.tajo.ResourceProtos.TajoHeartbeatRequest;
+import org.apache.tajo.ResourceProtos.TajoHeartbeatResponse;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.scheduler.SimpleFifoScheduler;
+import org.apache.tajo.master.scheduler.QuerySchedulingInfo;
import org.apache.tajo.plan.logical.LogicalRootNode;
import org.apache.tajo.querymaster.QueryJobEvent;
import org.apache.tajo.session.Session;
@@ -54,14 +56,13 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class QueryManager extends CompositeService {
private static final Log LOG = LogFactory.getLog(QueryManager.class.getName());
+ private static final String EMPTY_QM_HOSTNAME = "";
// TajoMaster Context
private final TajoMaster.MasterContext masterContext;
private AsyncDispatcher dispatcher;
- private SimpleFifoScheduler scheduler;
-
private final Map<QueryId, QueryInProgress> submittedQueries = Maps.newConcurrentMap();
private final Map<QueryId, QueryInProgress> runningQueries = Maps.newConcurrentMap();
@@ -85,7 +86,6 @@ public class QueryManager extends CompositeService {
this.dispatcher.register(QueryJobEvent.Type.class, new QueryJobManagerEventHandler());
- this.scheduler = new SimpleFifoScheduler(this);
} catch (Exception e) {
LOG.error("Failed to init service " + getName() + " by exception " + e, e);
}
@@ -95,18 +95,15 @@ public class QueryManager extends CompositeService {
@Override
public void serviceStop() throws Exception {
- synchronized(runningQueries) {
- for(QueryInProgress eachQueryInProgress: runningQueries.values()) {
- eachQueryInProgress.stopProgress();
- }
+ for(QueryInProgress eachQueryInProgress: runningQueries.values()) {
+ eachQueryInProgress.stopProgress();
}
- this.scheduler.stop();
+
super.serviceStop();
}
@Override
public void serviceStart() throws Exception {
- this.scheduler.start();
super.serviceStart();
}
@@ -115,28 +112,25 @@ public class QueryManager extends CompositeService {
}
public Collection<QueryInProgress> getSubmittedQueries() {
- synchronized (submittedQueries){
- return Collections.unmodifiableCollection(submittedQueries.values());
- }
+ return Collections.unmodifiableCollection(submittedQueries.values());
}
public Collection<QueryInProgress> getRunningQueries() {
- synchronized (runningQueries){
- return Collections.unmodifiableCollection(runningQueries.values());
- }
+ return Collections.unmodifiableCollection(runningQueries.values());
}
public synchronized Collection<QueryInfo> getFinishedQueries() {
+ Set<QueryInfo> result = Sets.newTreeSet();
+ synchronized (historyCache) {
+ result.addAll(historyCache.values());
+ }
+
try {
- Set<QueryInfo> result = Sets.newTreeSet();
result.addAll(this.masterContext.getHistoryReader().getQueries(null));
- synchronized (historyCache) {
- result.addAll(historyCache.values());
- }
return result;
} catch (Throwable e) {
LOG.error(e, e);
- return Lists.newArrayList();
+ return result;
}
}
@@ -171,6 +165,9 @@ public class QueryManager extends CompositeService {
return queryInProgress.getQueryInfo();
}
+ /**
+ * submit query to scheduler
+ */
public QueryInfo scheduleQuery(Session session, QueryContext queryContext, String sql,
String jsonExpr, LogicalRootNode plan)
throws Exception {
@@ -178,35 +175,29 @@ public class QueryManager extends CompositeService {
QueryInProgress queryInProgress = new QueryInProgress(masterContext, session, queryContext, queryId, sql,
jsonExpr, plan);
- synchronized (submittedQueries) {
- queryInProgress.getQueryInfo().setQueryMaster("");
- submittedQueries.put(queryInProgress.getQueryId(), queryInProgress);
- }
+ queryInProgress.getQueryInfo().setQueryMaster(EMPTY_QM_HOSTNAME);
+ submittedQueries.put(queryInProgress.getQueryId(), queryInProgress);
+ //TODO implement scheduler queue
+ QuerySchedulingInfo querySchedulingInfo = new QuerySchedulingInfo("default", queryContext.getUser(),
+ queryInProgress.getQueryId(), 1, queryInProgress.getQueryInfo().getStartTime());
- scheduler.addQuery(queryInProgress);
+ masterContext.getResourceManager().submitQuery(querySchedulingInfo);
return queryInProgress.getQueryInfo();
}
- public QueryInfo startQueryJob(QueryId queryId) throws Exception {
-
- QueryInProgress queryInProgress;
+ /**
+ * Can start query or not
+ */
+ public boolean startQueryJob(QueryId queryId, AllocationResourceProto allocation) {
- synchronized (submittedQueries) {
- queryInProgress = submittedQueries.remove(queryId);
- }
-
- synchronized (runningQueries) {
+ if (submittedQueries.get(queryId).allocateToQueryMaster(allocation)) {
+ QueryInProgress queryInProgress = submittedQueries.remove(queryId);
runningQueries.put(queryInProgress.getQueryId(), queryInProgress);
- }
-
- if (queryInProgress.startQueryMaster()) {
dispatcher.getEventHandler().handle(new QueryJobEvent(QueryJobEvent.Type.QUERY_MASTER_START,
queryInProgress.getQueryInfo()));
- } else {
- masterContext.getQueryJobManager().stopQuery(queryInProgress.getQueryId());
+ return true;
}
-
- return queryInProgress.getQueryInfo();
+ return false;
}
class QueryJobManagerEventHandler implements EventHandler<QueryJobEvent> {
@@ -221,10 +212,10 @@ public class QueryManager extends CompositeService {
}
if (event.getType() == QueryJobEvent.Type.QUERY_MASTER_START) {
- queryInProgress.submitQueryToMaster();
+ queryInProgress.submitToQueryMaster();
} else if (event.getType() == QueryJobEvent.Type.QUERY_JOB_KILL) {
- scheduler.removeQuery(queryInProgress.getQueryId());
+
queryInProgress.kill();
stopQuery(queryInProgress.getQueryId());
@@ -236,14 +227,10 @@ public class QueryManager extends CompositeService {
public QueryInProgress getQueryInProgress(QueryId queryId) {
QueryInProgress queryInProgress;
- synchronized (submittedQueries) {
- queryInProgress = submittedQueries.get(queryId);
- }
+ queryInProgress = submittedQueries.get(queryId);
if (queryInProgress == null) {
- synchronized (runningQueries) {
- queryInProgress = runningQueries.get(queryId);
- }
+ queryInProgress = runningQueries.get(queryId);
}
return queryInProgress;
}
@@ -253,13 +240,8 @@ public class QueryManager extends CompositeService {
QueryInProgress queryInProgress = getQueryInProgress(queryId);
if(queryInProgress != null) {
queryInProgress.stopProgress();
- synchronized(submittedQueries) {
- submittedQueries.remove(queryId);
- }
-
- synchronized(runningQueries) {
- runningQueries.remove(queryId);
- }
+ submittedQueries.remove(queryId);
+ runningQueries.remove(queryId);
QueryInfo queryInfo = queryInProgress.getQueryInfo();
synchronized (historyCache) {
@@ -304,8 +286,8 @@ public class QueryManager extends CompositeService {
return executedQuerySize.get();
}
- public synchronized QueryCoordinatorProtocol.TajoHeartbeatResponse.ResponseCommand queryHeartbeat(
- QueryCoordinatorProtocol.TajoHeartbeat queryHeartbeat) {
+ public synchronized TajoHeartbeatResponse.ResponseCommand queryHeartbeat(
+ TajoHeartbeatRequest queryHeartbeat) {
QueryInProgress queryInProgress = getQueryInProgress(new QueryId(queryHeartbeat.getQueryId()));
if(queryInProgress == null) {
return null;
@@ -317,7 +299,7 @@ public class QueryManager extends CompositeService {
return null;
}
- private QueryInfo makeQueryInfoFromHeartbeat(QueryCoordinatorProtocol.TajoHeartbeat queryHeartbeat) {
+ private QueryInfo makeQueryInfoFromHeartbeat(ResourceProtos.TajoHeartbeatRequest queryHeartbeat) {
QueryInfo queryInfo = new QueryInfo(new QueryId(queryHeartbeat.getQueryId()));
WorkerConnectionInfo connectionInfo = new WorkerConnectionInfo(queryHeartbeat.getConnectionInfo());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
deleted file mode 100644
index 2aac005..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.container.TajoContainer;
-import org.apache.tajo.master.container.TajoContainerId;
-import org.apache.tajo.master.event.TaskFatalErrorEvent;
-import org.apache.tajo.master.rm.TajoWorkerContainer;
-import org.apache.tajo.master.rm.TajoWorkerContainerId;
-import org.apache.tajo.plan.serder.PlanProto;
-import org.apache.tajo.querymaster.QueryMasterTask;
-import org.apache.tajo.rpc.NettyClientBase;
-import org.apache.tajo.rpc.NullCallback;
-import org.apache.tajo.rpc.RpcClientManager;
-import org.apache.tajo.service.ServiceTracker;
-import org.apache.tajo.worker.TajoWorker;
-
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class TajoContainerProxy extends ContainerProxy {
- private final QueryContext queryContext;
- private final TajoWorker.WorkerContext workerContext;
- private final String planJson;
-
- public TajoContainerProxy(QueryMasterTask.QueryMasterTaskContext context,
- Configuration conf, TajoContainer container,
- QueryContext queryContext, ExecutionBlockId executionBlockId, String planJson) {
- super(context, conf, executionBlockId, container);
- this.queryContext = queryContext;
- this.workerContext = context.getQueryMasterContext().getWorkerContext();
- this.planJson = planJson;
- }
-
- @Override
- public synchronized void launch(ContainerLaunchContext containerLaunchContext) {
- context.getResourceAllocator().addContainer(containerId, this);
-
- this.hostName = container.getNodeId().getHost();
- this.port = ((TajoWorkerContainer)container).getWorkerResource().getConnectionInfo().getPullServerPort();
- this.state = ContainerState.RUNNING;
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Launch Container:" + executionBlockId + "," + containerId.getId() + "," +
- container.getId() + "," + container.getNodeId() + ", pullServer=" + port);
- }
-
- assignExecutionBlock(executionBlockId, container);
- }
-
- /**
- * It sends a kill RPC request to a corresponding worker.
- *
- * @param taskAttemptId The TaskAttemptId to be killed.
- */
- public void killTaskAttempt(TaskAttemptId taskAttemptId) {
- NettyClientBase tajoWorkerRpc = null;
- try {
- InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
- tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true);
- TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
- tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), NullCallback.get());
- } catch (Throwable e) {
- /* Worker RPC failure */
- context.getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e.getMessage()));
- }
- }
-
- private void assignExecutionBlock(ExecutionBlockId executionBlockId, TajoContainer container) {
- NettyClientBase tajoWorkerRpc;
- try {
-
- InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
- tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true);
- TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
-
- PlanProto.ShuffleType shuffleType =
- context.getQuery().getStage(executionBlockId).getDataChannel().getShuffleType();
-
- TajoWorkerProtocol.RunExecutionBlockRequestProto request =
- TajoWorkerProtocol.RunExecutionBlockRequestProto.newBuilder()
- .setExecutionBlockId(executionBlockId.getProto())
- .setQueryMaster(context.getQueryMasterContext().getWorkerContext().getConnectionInfo().getProto())
- .setNodeId(container.getNodeId().toString())
- .setContainerId(container.getId().toString())
- .setQueryOutputPath(context.getStagingDir().toString())
- .setQueryContext(queryContext.getProto())
- .setPlanJson(planJson)
- .setShuffleType(shuffleType)
- .build();
-
- tajoWorkerRpcClient.startExecutionBlock(null, request, NullCallback.get());
- } catch (Throwable e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- @Override
- public synchronized void stopContainer() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Release TajoWorker Resource: " + executionBlockId + "," + containerId + ", state:" + this.state);
- }
- if(isCompletelyDone()) {
- LOG.info("Container already stopped:" + containerId);
- return;
- }
- if(this.state == ContainerState.PREP) {
- this.state = ContainerState.KILLED_BEFORE_LAUNCH;
- } else {
- try {
- releaseWorkerResource(context, executionBlockId, Arrays.asList(containerId));
- context.getResourceAllocator().removeContainer(containerId);
- } catch (Throwable t) {
- // ignore the cleanup failure
- String message = "cleanup failed for container "
- + this.containerId + " : "
- + StringUtils.stringifyException(t);
- LOG.warn(message);
- } finally {
- this.state = ContainerState.DONE;
- }
- }
- }
-
- public static void releaseWorkerResource(QueryMasterTask.QueryMasterTaskContext context,
- ExecutionBlockId executionBlockId,
- List<TajoContainerId> containerIds) throws Exception {
- List<ContainerProtocol.TajoContainerIdProto> containerIdProtos =
- new ArrayList<ContainerProtocol.TajoContainerIdProto>();
-
- for(TajoContainerId eachContainerId: containerIds) {
- containerIdProtos.add(TajoWorkerContainerId.getContainerIdProto(eachContainerId));
- }
-
- RpcClientManager manager = RpcClientManager.getInstance();
- NettyClientBase tmClient = null;
-
- ServiceTracker serviceTracker = context.getQueryMasterContext().getWorkerContext().getServiceTracker();
- tmClient = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true);
-
- QueryCoordinatorProtocol.QueryCoordinatorProtocolService masterClientService = tmClient.getStub();
- masterClientService.releaseWorkerResource(null,
- QueryCoordinatorProtocol.WorkerResourceReleaseRequest.newBuilder()
- .setExecutionBlockId(executionBlockId.getProto())
- .addAllContainerIds(containerIdProtos)
- .build(),
- NullCallback.get());
-
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index e1e85dd..9327c59 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -21,14 +21,10 @@ package org.apache.tajo.master;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
@@ -42,8 +38,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.function.FunctionLoader;
import org.apache.tajo.function.FunctionSignature;
-import org.apache.tajo.master.rm.TajoWorkerResourceManager;
-import org.apache.tajo.master.rm.WorkerResourceManager;
+import org.apache.tajo.master.rm.TajoResourceManager;
import org.apache.tajo.metrics.CatalogMetricsGaugeSet;
import org.apache.tajo.metrics.WorkerResourceMetricsGaugeSet;
import org.apache.tajo.rpc.RpcChannelFactory;
@@ -68,7 +63,6 @@ import java.io.*;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
-import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
@@ -121,7 +115,7 @@ public class TajoMaster extends CompositeService {
private QueryCoordinatorService tajoMasterService;
private SessionManager sessionManager;
- private WorkerResourceManager resourceManager;
+ private TajoResourceManager resourceManager;
//Web Server
private StaticHttpServer webServer;
private TajoRestService restServer;
@@ -157,66 +151,59 @@ public class TajoMaster extends CompositeService {
}
@Override
- public void serviceInit(Configuration _conf) throws Exception {
- if (!(_conf instanceof TajoConf)) {
- throw new IllegalArgumentException("_conf should be a TajoConf type.");
- }
- this.systemConf = (TajoConf) _conf;
+ public void serviceInit(Configuration conf) throws Exception {
+
+ this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));
context = new MasterContext(systemConf);
clock = new SystemClock();
- try {
- RackResolver.init(systemConf);
+ RackResolver.init(systemConf);
- RpcClientManager rpcManager = RpcClientManager.getInstance();
- rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES));
- rpcManager.setTimeoutSeconds(
- systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS));
+ RpcClientManager rpcManager = RpcClientManager.getInstance();
+ rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES));
+ rpcManager.setTimeoutSeconds(
+ systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS));
- initResourceManager();
+ initResourceManager();
- this.dispatcher = new AsyncDispatcher();
- addIfService(dispatcher);
+ this.dispatcher = new AsyncDispatcher();
+ addIfService(dispatcher);
// check the system directory and create if they are not created.
checkAndInitializeSystemDirectories();
diagnoseTajoMaster();
- catalogServer = new CatalogServer(loadFunctions());
- addIfService(catalogServer);
- catalog = new LocalCatalogWrapper(catalogServer, systemConf);
+ catalogServer = new CatalogServer(loadFunctions());
+ addIfService(catalogServer);
+ catalog = new LocalCatalogWrapper(catalogServer, systemConf);
- sessionManager = new SessionManager(dispatcher);
- addIfService(sessionManager);
+ sessionManager = new SessionManager(dispatcher);
+ addIfService(sessionManager);
- globalEngine = new GlobalEngine(context);
- addIfService(globalEngine);
+ globalEngine = new GlobalEngine(context);
+ addIfService(globalEngine);
- queryManager = new QueryManager(context);
- addIfService(queryManager);
+ queryManager = new QueryManager(context);
+ addIfService(queryManager);
- tajoMasterClientService = new TajoMasterClientService(context);
- addIfService(tajoMasterClientService);
+ tajoMasterClientService = new TajoMasterClientService(context);
+ addIfService(tajoMasterClientService);
- tajoMasterService = new QueryCoordinatorService(context);
- addIfService(tajoMasterService);
-
- restServer = new TajoRestService(context);
- addIfService(restServer);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw e;
- }
+ tajoMasterService = new QueryCoordinatorService(context);
+ addIfService(tajoMasterService);
+ restServer = new TajoRestService(context);
+ addIfService(restServer);
+
// Try to start up all services in TajoMaster.
// If anyone is failed, the master prints out the errors and immediately should shutdowns
try {
super.serviceInit(systemConf);
} catch (Throwable t) {
t.printStackTrace();
- System.exit(1);
+ Runtime.getRuntime().halt(-1);
}
LOG.info("Tajo Master is initialized.");
}
@@ -235,10 +222,7 @@ public class TajoMaster extends CompositeService {
}
private void initResourceManager() throws Exception {
- Class<WorkerResourceManager> resourceManagerClass = (Class<WorkerResourceManager>)
- systemConf.getClass(ConfVars.RESOURCE_MANAGER_CLASS.varname, TajoWorkerResourceManager.class);
- Constructor<WorkerResourceManager> constructor = resourceManagerClass.getConstructor(MasterContext.class);
- resourceManager = constructor.newInstance(context);
+ resourceManager = new TajoResourceManager(context);
addIfService(resourceManager);
}
@@ -391,39 +375,19 @@ public class TajoMaster extends CompositeService {
}
@Override
- public void stop() {
- if (haService != null) {
- try {
- haService.delete();
- } catch (Exception e) {
- LOG.error(e, e);
- }
- }
-
- if (restServer != null) {
- try {
- restServer.stop();
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
+ public void serviceStop() throws Exception {
+ if (haService != null) haService.delete();
- if (webServer != null) {
- try {
- webServer.stop();
- } catch (Exception e) {
- LOG.error(e, e);
- }
- }
+ if (restServer != null) restServer.stop();
+
+ if (webServer != null) webServer.stop();
IOUtils.cleanup(LOG, catalogServer);
- if(systemMetrics != null) {
- systemMetrics.stop();
- }
+ if (systemMetrics != null) systemMetrics.stop();
- if(pauseMonitor != null) pauseMonitor.stop();
- super.stop();
+ if (pauseMonitor != null) pauseMonitor.stop();
+ super.serviceStop();
LOG.info("Tajo Master main thread exiting");
}
@@ -463,7 +427,7 @@ public class TajoMaster extends CompositeService {
return queryManager;
}
- public WorkerResourceManager getResourceManager() {
+ public TajoResourceManager getResourceManager() {
return resourceManager;
}
@@ -601,7 +565,7 @@ public class TajoMaster extends CompositeService {
try {
TajoMaster master = new TajoMaster();
- TajoConf conf = new TajoConf(new YarnConfiguration());
+ TajoConf conf = new TajoConf();
master.init(conf);
master.start();
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 7dbe815..e0332d5 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -47,8 +47,7 @@ import org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolServ
import org.apache.tajo.master.TajoMaster.MasterContext;
import org.apache.tajo.master.exec.NonForwardQueryResultFileScanner;
import org.apache.tajo.master.exec.NonForwardQueryResultScanner;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
+import org.apache.tajo.master.rm.NodeStatus;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.PartitionedTableScanNode;
import org.apache.tajo.plan.logical.ScanNode;
@@ -95,32 +94,27 @@ public class TajoMasterClientService extends AbstractService {
}
@Override
- public void start() {
+ public void serviceStart() throws Exception {
// start the rpc server
String confClientServiceAddr = conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(confClientServiceAddr);
int workerNum = conf.getIntVar(ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM);
- try {
- server = new BlockingRpcServer(TajoMasterClientProtocol.class, clientHandler, initIsa, workerNum);
- } catch (Exception e) {
- LOG.error(e);
- throw new RuntimeException(e);
- }
+ server = new BlockingRpcServer(TajoMasterClientProtocol.class, clientHandler, initIsa, workerNum);
server.start();
bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
this.conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddress));
+ super.serviceStart();
LOG.info("Instantiated TajoMasterClientService at " + this.bindAddress);
- super.start();
}
@Override
- public void stop() {
+ public void serviceStop() throws Exception {
if (server != null) {
server.shutdown();
}
- super.stop();
+ super.serviceStop();
}
public InetSocketAddress getBindAddress() {
@@ -590,7 +584,7 @@ public class TajoMasterClientService extends AbstractService {
QueryManager queryManager = context.getQueryJobManager();
QueryInProgress queryInProgress = queryManager.getQueryInProgress(queryId);
- QueryInfo queryInfo = null;
+ QueryInfo queryInfo;
if (queryInProgress == null) {
queryInfo = context.getQueryJobManager().getFinishedQuery(queryId);
} else {
@@ -610,9 +604,6 @@ public class TajoMasterClientService extends AbstractService {
return builder.build();
}
- /**
- * It is invoked by TajoContainerProxy.
- */
@Override
public BoolProto killQuery(RpcController controller, QueryIdRequest request) throws ServiceException {
try {
@@ -642,31 +633,20 @@ public class TajoMasterClientService extends AbstractService {
context.getSessionManager().touch(request.getSessionId().getId());
GetClusterInfoResponse.Builder builder= GetClusterInfoResponse.newBuilder();
- Map<Integer, Worker> workers = context.getResourceManager().getWorkers();
-
- List<Integer> wokerKeys = new ArrayList<Integer>(workers.keySet());
- Collections.sort(wokerKeys);
-
- WorkerResourceInfo.Builder workerBuilder
- = WorkerResourceInfo.newBuilder();
-
- for(Worker worker: workers.values()) {
- WorkerResource workerResource = worker.getResource();
-
- workerBuilder.setConnectionInfo(worker.getConnectionInfo().getProto());
- workerBuilder.setDiskSlots(workerResource.getDiskSlots());
- workerBuilder.setCpuCoreSlots(workerResource.getCpuCoreSlots());
- workerBuilder.setMemoryMB(workerResource.getMemoryMB());
- workerBuilder.setLastHeartbeat(worker.getLastHeartbeatTime());
- workerBuilder.setUsedMemoryMB(workerResource.getUsedMemoryMB());
- workerBuilder.setUsedCpuCoreSlots(workerResource.getUsedCpuCoreSlots());
- workerBuilder.setUsedDiskSlots(workerResource.getUsedDiskSlots());
- workerBuilder.setWorkerStatus(worker.getState().toString());
- workerBuilder.setMaxHeap(workerResource.getMaxHeap());
- workerBuilder.setFreeHeap(workerResource.getFreeHeap());
- workerBuilder.setTotalHeap(workerResource.getTotalHeap());
- workerBuilder.setNumRunningTasks(workerResource.getNumRunningTasks());
- workerBuilder.setNumQueryMasterTasks(workerResource.getNumQueryMasterTasks());
+ List<NodeStatus> nodeStatusList = new ArrayList<NodeStatus>(context.getResourceManager().getRMContext().getNodes().values());
+ Collections.sort(nodeStatusList);
+
+ WorkerResourceInfo.Builder workerBuilder = WorkerResourceInfo.newBuilder();
+
+ for(NodeStatus nodeStatus : nodeStatusList) {
+ workerBuilder.setConnectionInfo(nodeStatus.getConnectionInfo().getProto());
+ workerBuilder.setAvailableResource(nodeStatus.getAvailableResource().getProto());
+ workerBuilder.setTotalResource(nodeStatus.getTotalResourceCapability().getProto());
+
+ workerBuilder.setLastHeartbeat(nodeStatus.getLastHeartbeatTime());
+ workerBuilder.setWorkerStatus(nodeStatus.getState().toString());
+ workerBuilder.setNumRunningTasks(nodeStatus.getNumRunningTasks());
+ workerBuilder.setNumQueryMasterTasks(nodeStatus.getNumRunningQueryMaster());
builder.addWorkerList(workerBuilder.build());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
deleted file mode 100644
index c1c6522..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
-import org.apache.tajo.master.container.TajoContainer;
-
-import java.util.Collection;
-
-public class TaskRunnerGroupEvent extends AbstractEvent<EventType> {
- public enum EventType {
- CONTAINER_REMOTE_LAUNCH,
- CONTAINER_REMOTE_CLEANUP
- }
-
- protected final ExecutionBlockId executionBlockId;
- protected final Collection<TajoContainer> containers;
- public TaskRunnerGroupEvent(EventType eventType,
- ExecutionBlockId executionBlockId,
- Collection<TajoContainer> containers) {
- super(eventType);
- this.executionBlockId = executionBlockId;
- this.containers = containers;
- }
-
- public Collection<TajoContainer> getContainers() {
- return containers;
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerLauncher.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerLauncher.java b/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerLauncher.java
deleted file mode 100644
index 9086e65..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerLauncher.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master;
-
-import org.apache.hadoop.yarn.event.EventHandler;
-
-public interface TaskRunnerLauncher extends EventHandler<TaskRunnerGroupEvent> {
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java b/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java
index 78d4978..0159a21 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/cluster/WorkerConnectionInfo.java
@@ -111,6 +111,10 @@ public class WorkerConnectionInfo implements ProtoObject<WorkerConnectionInfoPro
return this.getHost() + ":" + this.getPeerRpcPort();
}
+ public String getHostAndQMPort() {
+ return this.getHost() + ":" + this.getQueryMasterPort();
+ }
+
@Override
public WorkerConnectionInfoProto getProto() {
WorkerConnectionInfoProto.Builder builder = WorkerConnectionInfoProto.newBuilder();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
deleted file mode 100644
index 77562b5..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
-import org.apache.hadoop.yarn.api.records.*;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
- *
- * <p><code>TajoContainer</code> represents an allocated resource in the cluster.
- * </p>
- *
- * <p>The <code>ResourceManager</code> is the sole authority to allocate any
- * <code>TajoContainer</code> to applications. The allocated <code>TajoContainer</code>
- * is always on a single node and has a unique {@link org.apache.tajo.master.container.TajoContainerId}. It has
- * a specific amount of {@link org.apache.hadoop.yarn.api.records.Resource} allocated.</p>
- *
- * <p>It includes details such as:
- * <ul>
- * <li>{@link org.apache.tajo.master.container.TajoContainerId} for the container, which is globally unique.</li>
- * <li>
- * {@link org.apache.hadoop.yarn.api.records.NodeId} of the node on which it is allocated.
- * </li>
- * <li>HTTP uri of the node.</li>
- * <li>{@link org.apache.hadoop.yarn.api.records.Resource} allocated to the container.</li>
- * <li>{@link org.apache.hadoop.yarn.api.records.Priority} at which the container was allocated.</li>
- * <li>
- * TajoContainer {@link org.apache.hadoop.yarn.api.records.Token} of the container, used to securely verify
- * authenticity of the allocation.
- * </li>
- * </ul>
- * </p>
- *
- * <p>Typically, an <code>ApplicationMaster</code> receives the
- * <code>TajoContainer</code> from the <code>ResourceManager</code> during
- * resource-negotiation and then talks to the <code>NodeManager</code> to
- * start/stop containers.</p>
- *
- * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
- * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
- * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)
- */
-@Public
-@Stable
-public abstract class TajoContainer implements Comparable<TajoContainer> {
-
- @Private
- @Unstable
- public static TajoContainer newInstance(TajoContainerId containerId, NodeId nodeId,
- String nodeHttpAddress, Resource resource, Priority priority,
- Token containerToken) {
- TajoContainer container = Records.newRecord(TajoContainer.class);
- container.setId(containerId);
- container.setNodeId(nodeId);
- container.setNodeHttpAddress(nodeHttpAddress);
- container.setResource(resource);
- container.setPriority(priority);
- container.setContainerToken(containerToken);
- return container;
- }
-
- /**
- * Get the globally unique identifier for the container.
- * @return globally unique identifier for the container
- */
- @Public
- @Stable
- public abstract TajoContainerId getId();
-
- @Private
- @Unstable
- public abstract void setId(TajoContainerId id);
-
- /**
- * Get the identifier of the node on which the container is allocated.
- * @return identifier of the node on which the container is allocated
- */
- @Public
- @Stable
- public abstract NodeId getNodeId();
-
- @Private
- @Unstable
- public abstract void setNodeId(NodeId nodeId);
-
- /**
- * Get the http uri of the node on which the container is allocated.
- * @return http uri of the node on which the container is allocated
- */
- @Public
- @Stable
- public abstract String getNodeHttpAddress();
-
- @Private
- @Unstable
- public abstract void setNodeHttpAddress(String nodeHttpAddress);
-
- /**
- * Get the <code>Resource</code> allocated to the container.
- * @return <code>Resource</code> allocated to the container
- */
- @Public
- @Stable
- public abstract Resource getResource();
-
- @Private
- @Unstable
- public abstract void setResource(Resource resource);
-
- /**
- * Get the <code>Priority</code> at which the <code>TajoContainer</code> was
- * allocated.
- * @return <code>Priority</code> at which the <code>TajoContainer</code> was
- * allocated
- */
- @Public
- @Stable
- public abstract Priority getPriority();
-
- @Private
- @Unstable
- public abstract void setPriority(Priority priority);
-
- /**
- * Get the <code>TajoContainerToken</code> for the container.
- * <p><code>TajoContainerToken</code> is the security token used by the framework
- * to verify authenticity of any <code>TajoContainer</code>.</p>
- *
- * <p>The <code>ResourceManager</code>, on container allocation provides a
- * secure token which is verified by the <code>NodeManager</code> on
- * container launch.</p>
- *
- * <p>Applications do not need to care about <code>TajoContainerToken</code>, they
- * are transparently handled by the framework - the allocated
- * <code>TajoContainer</code> includes the <code>TajoContainerToken</code>.</p>
- *
- * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
- * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
- *
- * @return <code>TajoContainerToken</code> for the container
- */
- @Public
- @Stable
- public abstract Token getContainerToken();
-
- @Private
- @Unstable
- public abstract void setContainerToken(Token containerToken);
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
deleted file mode 100644
index 7bc27c6..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-import java.text.NumberFormat;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
- *
- * <p><code>TajoContainerId</code> represents a globally unique identifier
- * for a {@link org.apache.tajo.master.container.TajoContainer} in the cluster.</p>
- */
-@Public
-@Stable
-public abstract class TajoContainerId implements Comparable<TajoContainerId>{
-
- @Private
- @Unstable
- public static TajoContainerId newInstance(ApplicationAttemptId appAttemptId,
- int containerId) {
- TajoContainerId id = new TajoContainerIdPBImpl();
- id.setId(containerId);
- id.setApplicationAttemptId(appAttemptId);
- id.build();
- return id;
- }
-
- /**
- * Get the <code>ApplicationAttemptId</code> of the application to which the
- * <code>Container</code> was assigned.
- * <p>
- * Note: If containers are kept alive across application attempts via
- * {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
- * the <code>TajoContainerId</code> does not necessarily contain the current
- * running application attempt's <code>ApplicationAttemptId</code> This
- * container can be allocated by previously exited application attempt and
- * managed by the current running attempt thus have the previous application
- * attempt's <code>ApplicationAttemptId</code>.
- * </p>
- *
- * @return <code>ApplicationAttemptId</code> of the application to which the
- * <code>Container</code> was assigned
- */
- @Public
- @Stable
- public abstract ApplicationAttemptId getApplicationAttemptId();
-
- @Private
- @Unstable
- protected abstract void setApplicationAttemptId(ApplicationAttemptId atId);
-
- /**
- * Get the identifier of the <code>TajoContainerId</code>.
- * @return identifier of the <code>TajoContainerId</code>
- */
- @Public
- @Stable
- public abstract int getId();
-
- @Private
- @Unstable
- protected abstract void setId(int id);
-
-
- // TODO: fail the app submission if attempts are more than 10 or something
- private static final ThreadLocal<NumberFormat> appAttemptIdFormat =
- new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(2);
- return fmt;
- }
- };
- // TODO: Why thread local?
- // ^ NumberFormat instances are not threadsafe
- private static final ThreadLocal<NumberFormat> containerIdFormat =
- new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(6);
- return fmt;
- }
- };
-
- @Override
- public int hashCode() {
- // Generated by eclipse.
- final int prime = 435569;
- int result = 7507;
- result = prime * result + getId();
- result = prime * result + getApplicationAttemptId().hashCode();
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TajoContainerId other = (TajoContainerId) obj;
- if (!this.getApplicationAttemptId().equals(other.getApplicationAttemptId()))
- return false;
- if (this.getId() != other.getId())
- return false;
- return true;
- }
-
- @Override
- public int compareTo(TajoContainerId other) {
- if (this.getApplicationAttemptId().compareTo(
- other.getApplicationAttemptId()) == 0) {
- return this.getId() - other.getId();
- } else {
- return this.getApplicationAttemptId().compareTo(
- other.getApplicationAttemptId());
- }
-
- }
-
- @Override
- public String toString() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(4);
-
- StringBuilder sb = new StringBuilder();
- sb.append("container_");
- ApplicationId appId = getApplicationAttemptId().getApplicationId();
- sb.append(appId.getClusterTimestamp()).append("_");
- sb.append(fmt.format(appId.getId()))
- .append("_");
- sb.append(
- appAttemptIdFormat.get().format(
- getApplicationAttemptId().getAttemptId())).append("_");
- sb.append(containerIdFormat.get().format(getId()));
- return sb.toString();
- }
-
- protected abstract void build();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
deleted file mode 100644
index cf9e012..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
-
-import com.google.common.base.Preconditions;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.master.container.TajoContainerId;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
- *
- */
-@Private
-@Unstable
-public class TajoContainerIdPBImpl extends TajoContainerId {
- ContainerProtocol.TajoContainerIdProto proto = null;
- ContainerProtocol.TajoContainerIdProto.Builder builder = null;
- private ApplicationAttemptId applicationAttemptId = null;
-
- public TajoContainerIdPBImpl() {
- builder = ContainerProtocol.TajoContainerIdProto.newBuilder();
- }
-
- public TajoContainerIdPBImpl(ContainerProtocol.TajoContainerIdProto proto) {
- this.proto = proto;
- this.applicationAttemptId = convertFromProtoFormat(proto.getAppAttemptId());
- }
-
- public ContainerProtocol.TajoContainerIdProto getProto() {
- return proto;
- }
-
- @Override
- public int getId() {
- Preconditions.checkNotNull(proto);
- return proto.getId();
- }
-
- @Override
- protected void setId(int id) {
- Preconditions.checkNotNull(builder);
- builder.setId((id));
- }
-
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return this.applicationAttemptId;
- }
-
- @Override
- protected void setApplicationAttemptId(ApplicationAttemptId atId) {
- if (atId != null) {
- Preconditions.checkNotNull(builder);
- builder.setAppAttemptId(convertToProtoFormat(atId));
- }
- this.applicationAttemptId = atId;
- }
-
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(
- ApplicationAttemptIdProto p) {
- return new ApplicationAttemptIdPBImpl(p);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(
- ApplicationAttemptId t) {
- return ((ApplicationAttemptIdPBImpl)t).getProto();
- }
-
- @Override
- protected void build() {
- proto = builder.build();
- builder = null;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
deleted file mode 100644
index 88c4823..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tajo.master.container;
-
-
-import static org.apache.hadoop.yarn.util.StringHelper._split;
-
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
- *
- * This class contains a set of utilities which help converting data structures
- * from/to 'serializableFormat' to/from hadoop/nativejava data structures.
- *
- */
-@Private
-public class TajoConverterUtils {
-
- public static final String CONTAINER_PREFIX = "container";
-
- private static ApplicationAttemptId toApplicationAttemptId(
- Iterator<String> it) throws NumberFormatException {
- ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
- Integer.parseInt(it.next()));
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
- return appAttemptId;
- }
-
- public static String toString(TajoContainerId cId) {
- return cId == null ? null : cId.toString();
- }
-
- public static TajoContainerId toTajoContainerId(String containerIdStr) {
- Iterator<String> it = _split(containerIdStr).iterator();
- if (!it.next().equals(CONTAINER_PREFIX)) {
- throw new IllegalArgumentException("Invalid TajoContainerId prefix: "
- + containerIdStr);
- }
- try {
- ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
- TajoContainerId containerId =
- TajoContainerId.newInstance(appAttemptID, Integer.parseInt(it.next()));
- return containerId;
- } catch (NumberFormatException n) {
- throw new IllegalArgumentException("Invalid TajoContainerId: "
- + containerIdStr, n);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocationEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocationEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocationEvent.java
deleted file mode 100644
index c3a9a59..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocationEvent.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.ExecutionBlockId;
-
-public class ContainerAllocationEvent extends AbstractEvent<ContainerAllocatorEventType> {
-
- private final ExecutionBlockId executionBlockId;
- private final Priority priority;
- private final Resource resource;
- private final boolean isLeafQuery;
- private final int requiredNum;
- private final float progress;
-
- public ContainerAllocationEvent(ContainerAllocatorEventType eventType,
- ExecutionBlockId executionBlockId,
- Priority priority,
- Resource resource,
- int requiredNum,
- boolean isLeafQuery, float progress) {
- super(eventType);
- this.executionBlockId = executionBlockId;
- this.priority = priority;
- this.resource = resource;
- this.requiredNum = requiredNum;
- this.isLeafQuery = isLeafQuery;
- this.progress = progress;
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
- }
-
- public Priority getPriority() {
- return priority;
- }
-
- public int getRequiredNum() {
- return requiredNum;
- }
-
- public boolean isLeafQuery() {
- return isLeafQuery;
- }
-
- public Resource getCapability() {
- return resource;
- }
-
- public float getProgress() {
- return progress;
- }
-
- public Resource getResource() {
- return resource;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
deleted file mode 100644
index 183aeb5..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-public enum ContainerAllocatorEventType {
- // producer: TaskAttempt, consumer: ContainerAllocator
- CONTAINER_REQ,
- CONTAINER_DEALLOCATE,
- CONTAINER_FAILED
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerEvent.java
deleted file mode 100644
index 723ac1a..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.master.event.ContainerEvent.EventType;
-
-public class ContainerEvent extends AbstractEvent<EventType> {
- public enum EventType {
- CONTAINER_LAUNCHED,
- CONTAINER_STOPPED
- }
-
- private final ContainerId cId;
-
- public ContainerEvent(EventType eventType, ContainerId cId) {
- super(eventType);
- this.cId = cId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/GrouppedContainerAllocatorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/GrouppedContainerAllocatorEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/GrouppedContainerAllocatorEvent.java
deleted file mode 100644
index c34b174..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/GrouppedContainerAllocatorEvent.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.tajo.ExecutionBlockId;
-
-import java.util.Map;
-
-public class GrouppedContainerAllocatorEvent
- extends ContainerAllocationEvent {
- private final Map<String, Integer> requestMap;
-
- public GrouppedContainerAllocatorEvent(ContainerAllocatorEventType eventType,
- ExecutionBlockId executionBlockId,
- Priority priority,
- Resource resource,
- Map<String, Integer> requestMap,
- boolean isLeafQuery, float progress) {
- super(eventType, executionBlockId, priority,
- resource, requestMap.size(), isLeafQuery, progress);
- this.requestMap = requestMap;
- }
-
- public Map<String, Integer> getRequestMap() {
- return this.requestMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
index 5cf9887..5a36ba9 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
@@ -20,27 +20,26 @@ package org.apache.tajo.master.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.master.container.TajoContainerId;
/**
* This event is sent to a running TaskAttempt on a worker.
*/
public class LocalTaskEvent extends AbstractEvent<LocalTaskEventType> {
private final TaskAttemptId taskAttemptId;
- private final TajoContainerId containerId;
+ private final int workerId;
- public LocalTaskEvent(TaskAttemptId taskAttemptId, TajoContainerId containerId,
+ public LocalTaskEvent(TaskAttemptId taskAttemptId, int workerId,
LocalTaskEventType eventType) {
super(eventType);
this.taskAttemptId = taskAttemptId;
- this.containerId = containerId;
+ this.workerId = workerId;
}
public TaskAttemptId getTaskAttemptId() {
return taskAttemptId;
}
- public TajoContainerId getContainerId() {
- return containerId;
+ public int getWorkerId() {
+ return workerId;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/QueryStartEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryStartEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/QueryStartEvent.java
index 3a387fa..9ce7f09 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryStartEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/QueryStartEvent.java
@@ -21,6 +21,7 @@ package org.apache.tajo.master.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.QueryId;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.session.Session;
/**
@@ -36,15 +37,17 @@ public class QueryStartEvent extends AbstractEvent {
private final QueryContext queryContext;
private final String jsonExpr;
private final String logicalPlanJson;
+ private final NodeResource allocation;
public QueryStartEvent(QueryId queryId, Session session, QueryContext queryContext, String jsonExpr,
- String logicalPlanJson) {
+ String logicalPlanJson, NodeResource allocation) {
super(EventType.QUERY_START);
this.queryId = queryId;
this.session = session;
this.queryContext = queryContext;
this.jsonExpr = jsonExpr;
this.logicalPlanJson = logicalPlanJson;
+ this.allocation = allocation;
}
public QueryId getQueryId() {
@@ -67,6 +70,10 @@ public class QueryStartEvent extends AbstractEvent {
return logicalPlanJson;
}
+ public NodeResource getAllocation() {
+ return allocation;
+ }
+
@Override
public String toString() {
return getClass().getName() + "," + getType() + "," + queryId;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/StageContainerAllocationEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/StageContainerAllocationEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/StageContainerAllocationEvent.java
deleted file mode 100644
index 0d29e44..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/StageContainerAllocationEvent.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.master.container.TajoContainer;
-
-import java.util.List;
-
-public class StageContainerAllocationEvent extends StageEvent {
- private List<TajoContainer> allocatedContainer;
-
- public StageContainerAllocationEvent(final ExecutionBlockId id,
- List<TajoContainer> allocatedContainer) {
- super(id, StageEventType.SQ_CONTAINER_ALLOCATED);
- this.allocatedContainer = allocatedContainer;
- }
-
- public List<TajoContainer> getAllocatedContainer() {
- return this.allocatedContainer;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/StageEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/StageEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/event/StageEventType.java
index 763d426..d9beaa8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/StageEventType.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/StageEventType.java
@@ -26,9 +26,7 @@ public enum StageEventType {
// Producer: Query
SQ_INIT,
SQ_START,
- SQ_CONTAINER_ALLOCATED,
SQ_KILL,
- SQ_LAUNCH,
// Producer: Task
SQ_TASK_COMPLETED,
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/StageShuffleReportEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/StageShuffleReportEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/StageShuffleReportEvent.java
index 924fb59..8a3dcb0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/StageShuffleReportEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/StageShuffleReportEvent.java
@@ -19,20 +19,20 @@
package org.apache.tajo.master.event;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ResourceProtos.ExecutionBlockReport;
/**
* Event Class: From {@link org.apache.tajo.querymaster.QueryMasterManagerService} to Stage
*/
public class StageShuffleReportEvent extends StageEvent {
- private TajoWorkerProtocol.ExecutionBlockReport report;
+ private ExecutionBlockReport report;
- public StageShuffleReportEvent(ExecutionBlockId executionBlockId, TajoWorkerProtocol.ExecutionBlockReport report) {
+ public StageShuffleReportEvent(ExecutionBlockId executionBlockId, ExecutionBlockReport report) {
super(executionBlockId, StageEventType.SQ_SHUFFLE_REPORT);
this.report = report;
}
- public TajoWorkerProtocol.ExecutionBlockReport getReport() {
+ public ExecutionBlockReport getReport() {
return report;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
index 1611370..08ef805 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
@@ -20,23 +20,16 @@ package org.apache.tajo.master.event;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.container.TajoContainerId;
public class TaskAttemptAssignedEvent extends TaskAttemptEvent {
- private final TajoContainerId cId;
private final WorkerConnectionInfo workerConnectionInfo;
- public TaskAttemptAssignedEvent(TaskAttemptId id, TajoContainerId cId,
+ public TaskAttemptAssignedEvent(TaskAttemptId id,
WorkerConnectionInfo connectionInfo) {
super(id, TaskAttemptEventType.TA_ASSIGNED);
- this.cId = cId;
this.workerConnectionInfo = connectionInfo;
}
- public TajoContainerId getContainerId() {
- return cId;
- }
-
public WorkerConnectionInfo getWorkerConnectionInfo(){
return workerConnectionInfo;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEventType.java
index e35b154..f59b50b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEventType.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEventType.java
@@ -33,6 +33,7 @@ public enum TaskAttemptEventType {
//Producer:Scheduler
TA_ASSIGNED,
+ TA_ASSIGN_CANCEL,
TA_SCHEDULE_CANCELED,
//Producer:Scheduler
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
index 8c5f016..a9af288 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
@@ -19,7 +19,7 @@
package org.apache.tajo.master.event;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskStatusProto;
+import org.apache.tajo.ResourceProtos.TaskStatusProto;
public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent {
private final TaskStatusProto status;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
index 5a016fb..6799ce1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
@@ -20,9 +20,8 @@ package org.apache.tajo.master.event;
import com.google.protobuf.RpcCallback;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
import org.apache.tajo.querymaster.TaskAttempt;
-import org.apache.tajo.master.container.TajoContainerId;
public class TaskAttemptToSchedulerEvent extends TaskSchedulerEvent {
private final TaskAttemptScheduleContext context;
@@ -44,30 +43,13 @@ public class TaskAttemptToSchedulerEvent extends TaskSchedulerEvent {
}
public static class TaskAttemptScheduleContext {
- private TajoContainerId containerId;
private String host;
- private RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback;
+ private RpcCallback<TaskRequestProto> callback;
public TaskAttemptScheduleContext() {
}
- public TaskAttemptScheduleContext(TajoContainerId containerId,
- String host,
- RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback) {
- this.containerId = containerId;
- this.host = host;
- this.callback = callback;
- }
-
- public TajoContainerId getContainerId() {
- return containerId;
- }
-
- public void setContainerId(TajoContainerId containerId) {
- this.containerId = containerId;
- }
-
public String getHost() {
return host;
}
@@ -76,11 +58,11 @@ public class TaskAttemptToSchedulerEvent extends TaskSchedulerEvent {
this.host = host;
}
- public RpcCallback<TajoWorkerProtocol.TaskRequestProto> getCallback() {
+ public RpcCallback<TaskRequestProto> getCallback() {
return callback;
}
- public void setCallback(RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback) {
+ public void setCallback(RpcCallback<TaskRequestProto> callback) {
this.callback = callback;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
index 20204aa..66275b1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
@@ -19,7 +19,7 @@
package org.apache.tajo.master.event;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
+import org.apache.tajo.ResourceProtos.TaskCompletionReport;
public class TaskCompletionEvent extends TaskAttemptEvent {
private TaskCompletionReport report;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
index 03888bd..d50fcb8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
@@ -19,7 +19,7 @@
package org.apache.tajo.master.event;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskFatalErrorReport;
+import org.apache.tajo.ResourceProtos.TaskFatalErrorReport;
public class TaskFatalErrorEvent extends TaskAttemptEvent {
private final String message;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
index 3f72ed9..495eaf2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
@@ -18,13 +18,10 @@
package org.apache.tajo.master.event;
-import com.google.protobuf.RpcCallback;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto;
+import org.apache.tajo.ResourceProtos.AllocationResourceProto;
import org.apache.tajo.master.event.TaskRequestEvent.TaskRequestEventType;
-import org.apache.tajo.master.container.TajoContainerId;
public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
@@ -32,36 +29,28 @@ public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
TASK_REQ
}
- private final int workerId;
- private final TajoContainerId containerId;
+ private final AllocationResourceProto responseProto;
private final ExecutionBlockId executionBlockId;
-
- private final RpcCallback<TaskRequestProto> callback;
+ private final int workerId;
public TaskRequestEvent(int workerId,
- TajoContainerId containerId,
- ExecutionBlockId executionBlockId,
- RpcCallback<TaskRequestProto> callback) {
+ AllocationResourceProto responseProto,
+ ExecutionBlockId executionBlockId) {
super(TaskRequestEventType.TASK_REQ);
this.workerId = workerId;
- this.containerId = containerId;
+ this.responseProto = responseProto;
this.executionBlockId = executionBlockId;
- this.callback = callback;
- }
-
- public int getWorkerId() {
- return this.workerId;
- }
-
- public TajoContainerId getContainerId() {
- return this.containerId;
}
public ExecutionBlockId getExecutionBlockId() {
return executionBlockId;
}
- public RpcCallback<TajoWorkerProtocol.TaskRequestProto> getCallback() {
- return this.callback;
+ public AllocationResourceProto getResponseProto() {
+ return responseProto;
+ }
+
+ public int getWorkerId() {
+ return workerId;
}
}
[05/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/LegacyTaskImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/LegacyTaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/LegacyTaskImpl.java
deleted file mode 100644
index f97ce29..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/LegacyTaskImpl.java
+++ /dev/null
@@ -1,844 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import io.netty.handler.codec.http.QueryStringDecoder;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.TajoProtos;
-import org.apache.tajo.TajoProtos.TaskAttemptState;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.planner.physical.PhysicalExec;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.engine.query.TaskRequest;
-import org.apache.tajo.ipc.QueryMasterProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.*;
-import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.plan.function.python.TajoScriptEngine;
-import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
-import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.pullserver.TajoPullServerService;
-import org.apache.tajo.pullserver.retriever.FileChunk;
-import org.apache.tajo.rpc.NullCallback;
-import org.apache.tajo.storage.*;
-import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.util.NetUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.URI;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-
-import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
-
-@Deprecated
-public class LegacyTaskImpl implements Task {
- private static final Log LOG = LogFactory.getLog(LegacyTaskImpl.class);
- private static final float FETCHER_PROGRESS = 0.5f;
-
- private final TajoConf systemConf;
- private final QueryContext queryContext;
- private final ExecutionBlockContext executionBlockContext;
- private final String taskRunnerId;
-
- private final Path taskDir;
- private final TaskRequest request;
- private TaskAttemptContext context;
- private List<Fetcher> fetcherRunners;
- private LogicalNode plan;
- private final Map<String, TableDesc> descs = Maps.newHashMap();
- private PhysicalExec executor;
- private boolean interQuery;
- private Path inputTableBaseDir;
-
- private long startTime;
- private long finishTime;
-
- private final TableStats inputStats;
- private List<FileChunk> localChunks;
-
- // TODO - to be refactored
- private ShuffleType shuffleType = null;
- private Schema finalSchema = null;
- private TupleComparator sortComp = null;
-
- public LegacyTaskImpl(String taskRunnerId,
- Path baseDir,
- TaskAttemptId taskId,
- final ExecutionBlockContext executionBlockContext,
- final TaskRequest request) throws IOException {
- this(taskRunnerId, baseDir, taskId, executionBlockContext.getConf(), executionBlockContext, request);
- }
-
- public LegacyTaskImpl(String taskRunnerId,
- Path baseDir,
- TaskAttemptId taskId,
- TajoConf conf,
- final ExecutionBlockContext executionBlockContext,
- final TaskRequest request) throws IOException {
- this.taskRunnerId = taskRunnerId;
- this.request = request;
-
- this.systemConf = conf;
- this.queryContext = request.getQueryContext(systemConf);
- this.executionBlockContext = executionBlockContext;
- this.taskDir = StorageUtil.concatPath(baseDir,
- taskId.getTaskId().getId() + "_" + taskId.getId());
-
- this.context = new TaskAttemptContext(queryContext, executionBlockContext, taskId,
- request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir);
- this.context.setDataChannel(request.getDataChannel());
- this.context.setEnforcer(request.getEnforcer());
- this.context.setState(TaskAttemptState.TA_PENDING);
- this.inputStats = new TableStats();
- this.fetcherRunners = Lists.newArrayList();
- }
-
- public void initPlan() throws IOException {
- plan = LogicalNodeDeserializer.deserialize(queryContext, context.getEvalContext(), request.getPlan());
- LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN);
- if (scanNode != null) {
- for (LogicalNode node : scanNode) {
- ScanNode scan = (ScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
-
- LogicalNode [] partitionScanNode = PlannerUtil.findAllNodes(plan, NodeType.PARTITIONS_SCAN);
- if (partitionScanNode != null) {
- for (LogicalNode node : partitionScanNode) {
- PartitionedTableScanNode scan = (PartitionedTableScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
-
- interQuery = request.getProto().getInterQuery();
- if (interQuery) {
- context.setInterQuery();
- this.shuffleType = context.getDataChannel().getShuffleType();
-
- if (shuffleType == ShuffleType.RANGE_SHUFFLE) {
- SortNode sortNode = PlannerUtil.findTopNode(plan, NodeType.SORT);
- this.finalSchema = PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys());
- this.sortComp = new BaseTupleComparator(finalSchema, sortNode.getSortKeys());
- }
- } else {
- Path outFilePath = ((FileTablespace) TablespaceManager.get(queryContext.getStagingDir().toUri()).get())
- .getAppenderFilePath(getId(), queryContext.getStagingDir());
- LOG.info("Output File Path: " + outFilePath);
- context.setOutputPath(outFilePath);
- }
-
- this.localChunks = Collections.synchronizedList(new ArrayList<FileChunk>());
- LOG.info("==================================");
- LOG.info("* Stage " + request.getId() + " is initialized");
- LOG.info("* InterQuery: " + interQuery
- + (interQuery ? ", Use " + this.shuffleType + " shuffle" : "") +
- ", Fragments (num: " + request.getFragments().size() + ")" +
- ", Fetches (total:" + request.getFetches().size() + ") :");
-
- if(LOG.isDebugEnabled()) {
- for (FetchImpl f : request.getFetches()) {
- LOG.debug("Table Id: " + f.getName() + ", Simple URIs: " + f.getSimpleURIs());
- }
- }
- LOG.info("* Local task dir: " + taskDir);
- if(LOG.isDebugEnabled()) {
- LOG.debug("* plan:\n");
- LOG.debug(plan.toString());
- }
- LOG.info("==================================");
- }
-
- private void startScriptExecutors() throws IOException {
- for (TajoScriptEngine executor : context.getEvalContext().getAllScriptEngines()) {
- executor.start(systemConf);
- }
- }
-
- private void stopScriptExecutors() {
- for (TajoScriptEngine executor : context.getEvalContext().getAllScriptEngines()) {
- executor.shutdown();
- }
- }
-
- @Override
- public void init() throws IOException {
- initPlan();
- startScriptExecutors();
-
- if (context.getState() == TaskAttemptState.TA_PENDING) {
- // initialize a task temporal dir
- FileSystem localFS = executionBlockContext.getLocalFS();
- localFS.mkdirs(taskDir);
-
- if (request.getFetches().size() > 0) {
- inputTableBaseDir = localFS.makeQualified(
- executionBlockContext.getLocalDirAllocator().getLocalPathForWrite(
- getTaskAttemptDir(context.getTaskId()).toString(), systemConf));
- localFS.mkdirs(inputTableBaseDir);
- Path tableDir;
- for (String inputTable : context.getInputTables()) {
- tableDir = new Path(inputTableBaseDir, inputTable);
- if (!localFS.exists(tableDir)) {
- LOG.info("the directory is created " + tableDir.toUri());
- localFS.mkdirs(tableDir);
- }
- }
- }
- // for localizing the intermediate data
- fetcherRunners.addAll(getFetchRunners(context, request.getFetches()));
- }
- }
-
- private TaskAttemptId getId() {
- return context.getTaskId();
- }
-
- public String toString() {
- return "queryId: " + this.getId() + " status: " + context.getState();
- }
-
- @Override
- public boolean isStopped() {
- return context.isStopped();
- }
-
- @Override
- public TaskAttemptContext getTaskContext() {
- return context;
- }
-
- @Override
- public ExecutionBlockContext getExecutionBlockContext() {
- return executionBlockContext;
- }
-
- @Override
- public boolean hasFetchPhase() {
- return fetcherRunners.size() > 0;
- }
-
- @Override
- public void fetch() {
- ExecutorService executorService = executionBlockContext.getTaskRunner(taskRunnerId).getFetchLauncher();
- for (Fetcher f : fetcherRunners) {
- executorService.submit(new FetchRunner(context, f));
- }
- }
-
- @Override
- public void kill() {
- stopScriptExecutors();
- context.setState(TaskAttemptState.TA_KILLED);
- context.stop();
- }
-
- @Override
- public void abort() {
- stopScriptExecutors();
- context.setState(TajoProtos.TaskAttemptState.TA_FAILED);
- context.stop();
- }
-
- @Override
- public TaskStatusProto getReport() {
- TaskStatusProto.Builder builder = TaskStatusProto.newBuilder();
- builder.setWorkerName(executionBlockContext.getWorkerContext().getConnectionInfo().getHostAndPeerRpcPort());
- builder.setId(context.getTaskId().getProto())
- .setProgress(context.getProgress())
- .setState(context.getState());
-
- builder.setInputStats(reloadInputStats());
-
- if (context.getResultStats() != null) {
- builder.setResultStats(context.getResultStats().getProto());
- }
- return builder.build();
- }
-
- @Override
- public boolean isProgressChanged() {
- return context.isProgressChanged();
- }
-
- @Override
- public void updateProgress() {
- if(context != null && context.isStopped()){
- return;
- }
-
- if (executor != null && context.getProgress() < 1.0f) {
- context.setExecutorProgress(executor.getProgress());
- }
- }
-
- private CatalogProtos.TableStatsProto reloadInputStats() {
- synchronized(inputStats) {
- if (this.executor == null) {
- return inputStats.getProto();
- }
-
- TableStats executorInputStats = this.executor.getInputStats();
-
- if (executorInputStats != null) {
- inputStats.setValues(executorInputStats);
- }
- return inputStats.getProto();
- }
- }
-
- private TaskCompletionReport getTaskCompletionReport() {
- TaskCompletionReport.Builder builder = TaskCompletionReport.newBuilder();
- builder.setId(context.getTaskId().getProto());
-
- builder.setInputStats(reloadInputStats());
-
- if (context.hasResultStats()) {
- builder.setResultStats(context.getResultStats().getProto());
- } else {
- builder.setResultStats(new TableStats().getProto());
- }
-
- Iterator<Entry<Integer, String>> it = context.getShuffleFileOutputs();
- if (it.hasNext()) {
- do {
- Entry<Integer, String> entry = it.next();
- ShuffleFileOutput.Builder part = ShuffleFileOutput.newBuilder();
- part.setPartId(entry.getKey());
-
- // Set output volume
- if (context.getPartitionOutputVolume() != null) {
- for (Entry<Integer, Long> e : context.getPartitionOutputVolume().entrySet()) {
- if (entry.getKey().equals(e.getKey())) {
- part.setVolume(e.getValue().longValue());
- break;
- }
- }
- }
-
- builder.addShuffleFileOutputs(part.build());
- } while (it.hasNext());
- }
-
- return builder.build();
- }
-
- private void waitForFetch() throws InterruptedException, IOException {
- context.getFetchLatch().await();
- LOG.info(context.getTaskId() + " All fetches are done!");
- Collection<String> inputs = Lists.newArrayList(context.getInputTables());
-
- // Get all broadcasted tables
- Set<String> broadcastTableNames = new HashSet<String>();
- List<EnforceProperty> broadcasts = context.getEnforcer().getEnforceProperties(EnforceType.BROADCAST);
- if (broadcasts != null) {
- for (EnforceProperty eachBroadcast : broadcasts) {
- broadcastTableNames.add(eachBroadcast.getBroadcast().getTableName());
- }
- }
-
- // localize the fetched data and skip the broadcast table
- for (String inputTable: inputs) {
- if (broadcastTableNames.contains(inputTable)) {
- continue;
- }
- File tableDir = new File(context.getFetchIn(), inputTable);
- FileFragment[] frags = localizeFetchedData(tableDir, inputTable, descs.get(inputTable).getMeta());
- context.updateAssignedFragments(inputTable, frags);
- }
- }
-
- @Override
- public void run() throws Exception {
- startTime = System.currentTimeMillis();
- Throwable error = null;
- try {
- if(!context.isStopped()) {
- context.setState(TaskAttemptState.TA_RUNNING);
- if (context.hasFetchPhase()) {
- // If the fetch is still in progress, the query unit must wait for
- // complete.
- waitForFetch();
- context.setFetcherProgress(FETCHER_PROGRESS);
- updateProgress();
- }
-
- this.executor = executionBlockContext.getTQueryEngine().
- createPlan(context, plan);
- this.executor.init();
-
- while(!context.isStopped() && executor.next() != null) {
- }
- }
- } catch (Throwable e) {
- error = e ;
- LOG.error(e.getMessage(), e);
- stopScriptExecutors();
- context.stop();
- } finally {
- if (executor != null) {
- try {
- executor.close();
- reloadInputStats();
- } catch (IOException e) {
- LOG.error(e, e);
- }
- this.executor = null;
- }
-
- executionBlockContext.completedTasksNum.incrementAndGet();
- context.getHashShuffleAppenderManager().finalizeTask(getId());
-
- QueryMasterProtocol.QueryMasterProtocolService.Interface queryMasterStub = executionBlockContext.getStub();
- if (context.isStopped()) {
- context.setExecutorProgress(0.0f);
-
- if (context.getState() == TaskAttemptState.TA_KILLED) {
- queryMasterStub.statusUpdate(null, getReport(), NullCallback.get());
- executionBlockContext.killedTasksNum.incrementAndGet();
- } else {
- context.setState(TaskAttemptState.TA_FAILED);
- TaskFatalErrorReport.Builder errorBuilder =
- TaskFatalErrorReport.newBuilder()
- .setId(getId().getProto());
- if (error != null) {
- if (error.getMessage() == null) {
- errorBuilder.setErrorMessage(error.getClass().getCanonicalName());
- } else {
- errorBuilder.setErrorMessage(error.getMessage());
- }
- errorBuilder.setErrorTrace(ExceptionUtils.getStackTrace(error));
- }
-
- queryMasterStub.fatalError(null, errorBuilder.build(), NullCallback.get());
- executionBlockContext.failedTasksNum.incrementAndGet();
- }
- } else {
- // if successful
- context.stop();
- context.setProgress(1.0f);
- context.setState(TaskAttemptState.TA_SUCCEEDED);
- executionBlockContext.succeededTasksNum.incrementAndGet();
-
- TaskCompletionReport report = getTaskCompletionReport();
- queryMasterStub.done(null, report, NullCallback.get());
- }
- finishTime = System.currentTimeMillis();
- LOG.info(context.getTaskId() + " completed. " +
- "Worker's task counter - total:" + executionBlockContext.completedTasksNum.intValue() +
- ", succeeded: " + executionBlockContext.succeededTasksNum.intValue()
- + ", killed: " + executionBlockContext.killedTasksNum.intValue()
- + ", failed: " + executionBlockContext.failedTasksNum.intValue());
- }
- }
-
- @Override
- public void cleanup() {
- TaskHistory taskHistory = createTaskHistory();
- executionBlockContext.addTaskHistory(taskRunnerId, getId(), taskHistory);
- executionBlockContext.getTasks().remove(getId());
-
- fetcherRunners.clear();
- fetcherRunners = null;
- try {
- if(executor != null) {
- executor.close();
- executor = null;
- }
- } catch (IOException e) {
- LOG.fatal(e.getMessage(), e);
- }
-
- executionBlockContext.getWorkerContext().getTaskHistoryWriter().appendHistory(taskHistory);
- stopScriptExecutors();
- }
-
- public TaskHistory createTaskHistory() {
- TaskHistory taskHistory = null;
- try {
- taskHistory = new TaskHistory(context.getTaskId(), context.getState(), context.getProgress(),
- startTime, finishTime, reloadInputStats());
-
- if (context.getOutputPath() != null) {
- taskHistory.setOutputPath(context.getOutputPath().toString());
- }
-
- if (context.getWorkDir() != null) {
- taskHistory.setWorkingPath(context.getWorkDir().toString());
- }
-
- if (context.getResultStats() != null) {
- taskHistory.setOutputStats(context.getResultStats().getProto());
- }
-
- if (hasFetchPhase()) {
- taskHistory.setTotalFetchCount(fetcherRunners.size());
- int i = 0;
- FetcherHistoryProto.Builder builder = FetcherHistoryProto.newBuilder();
- for (Fetcher fetcher : fetcherRunners) {
- // TODO store the fetcher histories
- if (systemConf.getBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED)) {
- builder.setStartTime(fetcher.getStartTime());
- builder.setFinishTime(fetcher.getFinishTime());
- builder.setFileLength(fetcher.getFileLen());
- builder.setMessageReceivedCount(fetcher.getMessageReceiveCount());
- builder.setState(fetcher.getState());
-
- taskHistory.addFetcherHistory(builder.build());
- }
- if (fetcher.getState() == TajoProtos.FetcherState.FETCH_FINISHED) i++;
- }
- taskHistory.setFinishedFetchCount(i);
- }
- } catch (Exception e) {
- LOG.warn(e.getMessage(), e);
- }
-
- return taskHistory;
- }
-
- public int hashCode() {
- return context.hashCode();
- }
-
- public boolean equals(Object obj) {
- if (obj instanceof LegacyTaskImpl) {
- LegacyTaskImpl other = (LegacyTaskImpl) obj;
- return this.context.equals(other.context);
- }
- return false;
- }
-
- private FileFragment[] localizeFetchedData(File file, String name, TableMeta meta)
- throws IOException {
- Configuration c = new Configuration(systemConf);
- c.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "file:///");
- FileSystem fs = FileSystem.get(c);
- Path tablePath = new Path(file.getAbsolutePath());
-
- List<FileFragment> listTablets = new ArrayList<FileFragment>();
- FileFragment tablet;
-
- FileStatus[] fileLists = fs.listStatus(tablePath);
- for (FileStatus f : fileLists) {
- if (f.getLen() == 0) {
- continue;
- }
- tablet = new FileFragment(name, f.getPath(), 0l, f.getLen());
- listTablets.add(tablet);
- }
-
- // Special treatment for locally pseudo fetched chunks
- synchronized (localChunks) {
- for (FileChunk chunk : localChunks) {
- if (name.equals(chunk.getEbId())) {
- tablet = new FileFragment(name, new Path(chunk.getFile().getPath()), chunk.startOffset(), chunk.length());
- listTablets.add(tablet);
- LOG.info("One local chunk is added to listTablets");
- }
- }
- }
-
- FileFragment[] tablets = new FileFragment[listTablets.size()];
- listTablets.toArray(tablets);
-
- return tablets;
- }
-
- private class FetchRunner implements Runnable {
- private final TaskAttemptContext ctx;
- private final Fetcher fetcher;
- private int maxRetryNum;
-
- public FetchRunner(TaskAttemptContext ctx, Fetcher fetcher) {
- this.ctx = ctx;
- this.fetcher = fetcher;
- this.maxRetryNum = systemConf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_READ_RETRY_MAX_NUM);
- }
-
- @Override
- public void run() {
- int retryNum = 0;
- int retryWaitTime = 1000; //sec
-
- try { // for releasing fetch latch
- while(!context.isStopped() && retryNum < maxRetryNum) {
- if (retryNum > 0) {
- try {
- Thread.sleep(retryWaitTime);
- retryWaitTime = Math.min(10 * 1000, retryWaitTime * 2); // max 10 seconds
- } catch (InterruptedException e) {
- LOG.error(e);
- }
- LOG.warn("Retry on the fetch: " + fetcher.getURI() + " (" + retryNum + ")");
- }
- try {
- FileChunk fetched = fetcher.get();
- if (fetcher.getState() == TajoProtos.FetcherState.FETCH_FINISHED && fetched != null
- && fetched.getFile() != null) {
- if (fetched.fromRemote() == false) {
- localChunks.add(fetched);
- LOG.info("Add a new FileChunk to local chunk list");
- }
- break;
- }
- } catch (Throwable e) {
- LOG.error("Fetch failed: " + fetcher.getURI(), e);
- }
- retryNum++;
- }
- } finally {
- if(fetcher.getState() == TajoProtos.FetcherState.FETCH_FINISHED){
- fetcherFinished(ctx);
- } else {
- if (retryNum == maxRetryNum) {
- LOG.error("ERROR: the maximum retry (" + retryNum + ") on the fetch exceeded (" + fetcher.getURI() + ")");
- }
- stopScriptExecutors();
- context.stop(); // retry task
- ctx.getFetchLatch().countDown();
- }
- }
- }
- }
-
- @VisibleForTesting
- public static float adjustFetchProcess(int totalFetcher, int remainFetcher) {
- if (totalFetcher > 0) {
- return ((totalFetcher - remainFetcher) / (float) totalFetcher) * FETCHER_PROGRESS;
- } else {
- return 0.0f;
- }
- }
-
- private synchronized void fetcherFinished(TaskAttemptContext ctx) {
- int fetcherSize = fetcherRunners.size();
- if(fetcherSize == 0) {
- return;
- }
-
- ctx.getFetchLatch().countDown();
-
- int remainFetcher = (int) ctx.getFetchLatch().getCount();
- if (remainFetcher == 0) {
- context.setFetcherProgress(FETCHER_PROGRESS);
- } else {
- context.setFetcherProgress(adjustFetchProcess(fetcherSize, remainFetcher));
- }
- }
-
- private List<Fetcher> getFetchRunners(TaskAttemptContext ctx,
- List<FetchImpl> fetches) throws IOException {
-
- if (fetches.size() > 0) {
- Path inputDir = executionBlockContext.getLocalDirAllocator().
- getLocalPathToRead(getTaskAttemptDir(ctx.getTaskId()).toString(), systemConf);
-
- int i = 0;
- File storeDir;
- File defaultStoreFile;
- FileChunk storeChunk = null;
- List<Fetcher> runnerList = Lists.newArrayList();
-
- for (FetchImpl f : fetches) {
- storeDir = new File(inputDir.toString(), f.getName());
- if (!storeDir.exists()) {
- storeDir.mkdirs();
- }
-
- for (URI uri : f.getURIs()) {
- defaultStoreFile = new File(storeDir, "in_" + i);
- InetAddress address = InetAddress.getByName(uri.getHost());
-
- WorkerConnectionInfo conn = executionBlockContext.getWorkerContext().getConnectionInfo();
- if (NetUtils.isLocalAddress(address) && conn.getPullServerPort() == uri.getPort()) {
- boolean hasError = false;
- try {
- LOG.info("Try to get local file chunk at local host");
- storeChunk = getLocalStoredFileChunk(uri, systemConf);
- } catch (Throwable t) {
- hasError = true;
- }
-
- // When a range request is out of range, storeChunk will be NULL. This case is normal state.
- // So, we should skip and don't need to create storeChunk.
- if (storeChunk == null && !hasError) {
- continue;
- }
-
- if (storeChunk != null && storeChunk.getFile() != null && storeChunk.startOffset() > -1
- && hasError == false) {
- storeChunk.setFromRemote(false);
- } else {
- storeChunk = new FileChunk(defaultStoreFile, 0, -1);
- storeChunk.setFromRemote(true);
- }
- } else {
- storeChunk = new FileChunk(defaultStoreFile, 0, -1);
- storeChunk.setFromRemote(true);
- }
-
- // If we decide that intermediate data should be really fetched from a remote host, storeChunk
- // represents a complete file. Otherwise, storeChunk may represent a complete file or only a part of it
- storeChunk.setEbId(f.getName());
- Fetcher fetcher = new Fetcher(systemConf, uri, storeChunk);
- LOG.info("Create a new Fetcher with storeChunk:" + storeChunk.toString());
- runnerList.add(fetcher);
- i++;
- }
- }
- ctx.addFetchPhase(runnerList.size(), new File(inputDir.toString()));
- return runnerList;
- } else {
- return Lists.newArrayList();
- }
- }
-
- private FileChunk getLocalStoredFileChunk(URI fetchURI, TajoConf conf) throws IOException {
- // Parse the URI
- LOG.info("getLocalStoredFileChunk starts");
- final Map<String, List<String>> params = new QueryStringDecoder(fetchURI.toString()).parameters();
- final List<String> types = params.get("type");
- final List<String> qids = params.get("qid");
- final List<String> taskIdList = params.get("ta");
- final List<String> stageIds = params.get("sid");
- final List<String> partIds = params.get("p");
- final List<String> offsetList = params.get("offset");
- final List<String> lengthList = params.get("length");
-
- if (types == null || stageIds == null || qids == null || partIds == null) {
- LOG.error("Invalid URI - Required queryId, type, stage Id, and part id");
- return null;
- }
-
- if (qids.size() != 1 && types.size() != 1 || stageIds.size() != 1) {
- LOG.error("Invalid URI - Required qids, type, taskIds, stage Id, and part id");
- return null;
- }
-
- String queryId = qids.get(0);
- String shuffleType = types.get(0);
- String sid = stageIds.get(0);
- String partId = partIds.get(0);
-
- if (shuffleType.equals("r") && taskIdList == null) {
- LOG.error("Invalid URI - For range shuffle, taskId is required");
- return null;
- }
- List<String> taskIds = splitMaps(taskIdList);
-
- FileChunk chunk = null;
- long offset = (offsetList != null && !offsetList.isEmpty()) ? Long.parseLong(offsetList.get(0)) : -1L;
- long length = (lengthList != null && !lengthList.isEmpty()) ? Long.parseLong(lengthList.get(0)) : -1L;
-
- LOG.info("PullServer request param: shuffleType=" + shuffleType + ", sid=" + sid + ", partId=" + partId
- + ", taskIds=" + taskIdList);
-
- // The working directory of Tajo worker for each query, including stage
- String queryBaseDir = queryId.toString() + "/output" + "/" + sid + "/";
-
- // If the stage requires a range shuffle
- if (shuffleType.equals("r")) {
- String ta = taskIds.get(0);
- if (!executionBlockContext.getLocalDirAllocator().ifExists(queryBaseDir + ta + "/output/", conf)) {
- LOG.warn("Range shuffle - file not exist");
- return null;
- }
- Path path = executionBlockContext.getLocalFS().makeQualified(
- executionBlockContext.getLocalDirAllocator().getLocalPathToRead(queryBaseDir + ta + "/output/", conf));
- String startKey = params.get("start").get(0);
- String endKey = params.get("end").get(0);
- boolean last = params.get("final") != null;
-
- try {
- chunk = TajoPullServerService.getFileChunks(path, startKey, endKey, last);
- } catch (Throwable t) {
- LOG.error("getFileChunks() throws exception");
- return null;
- }
-
- // If the stage requires a hash shuffle or a scattered hash shuffle
- } else if (shuffleType.equals("h") || shuffleType.equals("s")) {
- int partParentId = HashShuffleAppenderManager.getPartParentId(Integer.parseInt(partId), (TajoConf) conf);
- String partPath = queryBaseDir + "hash-shuffle/" + partParentId + "/" + partId;
- if (!executionBlockContext.getLocalDirAllocator().ifExists(partPath, conf)) {
- LOG.warn("Hash shuffle or Scattered hash shuffle - file not exist: " + partPath);
- return null;
- }
- Path path = executionBlockContext.getLocalFS().makeQualified(
- executionBlockContext.getLocalDirAllocator().getLocalPathToRead(partPath, conf));
- File file = new File(path.toUri());
- long startPos = (offset >= 0 && length >= 0) ? offset : 0;
- long readLen = (offset >= 0 && length >= 0) ? length : file.length();
-
- if (startPos >= file.length()) {
- LOG.error("Start pos[" + startPos + "] great than file length [" + file.length() + "]");
- return null;
- }
- chunk = new FileChunk(file, startPos, readLen);
-
- } else {
- LOG.error("Unknown shuffle type");
- return null;
- }
-
- return chunk;
- }
-
- private List<String> splitMaps(List<String> mapq) {
- if (null == mapq) {
- return null;
- }
- final List<String> ret = new ArrayList<String>();
- for (String s : mapq) {
- Collections.addAll(ret, s.split(","));
- }
- return ret;
- }
-
- public static Path getTaskAttemptDir(TaskAttemptId quid) {
- Path workDir =
- StorageUtil.concatPath(ExecutionBlockContext.getBaseInputDir(quid.getTaskId().getExecutionBlockId()),
- String.valueOf(quid.getTaskId().getId()),
- String.valueOf(quid.getId()));
- return workDir;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/NodeResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeResourceManager.java
index e763d13..0580ebc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeResourceManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeResourceManager.java
@@ -30,35 +30,38 @@ import org.apache.tajo.resource.NodeResources;
import org.apache.tajo.storage.DiskUtil;
import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.event.*;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.tajo.ResourceProtos.*;
public class NodeResourceManager extends AbstractService implements EventHandler<NodeResourceEvent> {
private static final Log LOG = LogFactory.getLog(NodeResourceManager.class);
private final Dispatcher dispatcher;
- private final EventHandler taskEventHandler;
+ private final TajoWorker.WorkerContext workerContext;
+ private final AtomicInteger runningQueryMasters = new AtomicInteger(0);
private NodeResource totalResource;
private NodeResource availableResource;
private TajoConf tajoConf;
+ private boolean enableTest;
- public NodeResourceManager(Dispatcher dispatcher, EventHandler taskEventHandler) {
+ public NodeResourceManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext) {
super(NodeResourceManager.class.getName());
this.dispatcher = dispatcher;
- this.taskEventHandler = taskEventHandler;
+ this.workerContext = workerContext;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- this.tajoConf = (TajoConf)conf;
+ this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
this.totalResource = createWorkerResource(tajoConf);
this.availableResource = NodeResources.clone(totalResource);
this.dispatcher.register(NodeResourceEvent.EventType.class, this);
-
+ validateConf(tajoConf);
+ this.enableTest = conf.get(CommonTestingUtil.TAJO_TEST_KEY, "FALSE").equalsIgnoreCase("TRUE");
super.serviceInit(conf);
LOG.info("Initialized NodeResourceManager for " + totalResource);
}
@@ -66,63 +69,85 @@ public class NodeResourceManager extends AbstractService implements EventHandler
@Override
public void handle(NodeResourceEvent event) {
- if (event instanceof NodeResourceAllocateEvent) {
- NodeResourceAllocateEvent allocateEvent = (NodeResourceAllocateEvent) event;
- BatchAllocationResponseProto.Builder response = BatchAllocationResponseProto.newBuilder();
- for (TaskAllocationRequestProto request : allocateEvent.getRequest().getTaskRequestList()) {
- NodeResource resource = new NodeResource(request.getResource());
- if (allocate(resource)) {
- if(allocateEvent.getRequest().hasExecutionBlockRequest()){
- //send ExecutionBlock start event to TaskManager
- startExecutionBlock(allocateEvent.getRequest().getExecutionBlockRequest());
+ switch (event.getType()) {
+ case ALLOCATE: {
+ if (event.getResourceType() == NodeResourceEvent.ResourceType.TASK) {
+ // allocate task resource
+ NodeResourceAllocateEvent allocateEvent = TUtil.checkTypeAndGet(event, NodeResourceAllocateEvent.class);
+ BatchAllocationResponse.Builder response = BatchAllocationResponse.newBuilder();
+ for (TaskAllocationProto request : allocateEvent.getRequest().getTaskRequestList()) {
+ NodeResource resource = new NodeResource(request.getResource());
+ if (allocate(resource)) {
+ //send task start event to TaskExecutor
+ startTask(request.getTaskRequest(), resource);
+ } else {
+ // reject the exceeded requests
+ response.addCancellationTask(request);
+ }
+ }
+ allocateEvent.getCallback().run(response.build());
+
+ } else if (event.getResourceType() == NodeResourceEvent.ResourceType.QUERY_MASTER) {
+ QMResourceAllocateEvent allocateEvent = TUtil.checkTypeAndGet(event, QMResourceAllocateEvent.class);
+ // allocate query master resource
+
+ NodeResource resource = new NodeResource(allocateEvent.getRequest().getResource());
+ if (allocate(resource)) {
+ allocateEvent.getCallback().run(TajoWorker.TRUE_PROTO);
+ runningQueryMasters.incrementAndGet();
+ } else {
+ allocateEvent.getCallback().run(TajoWorker.FALSE_PROTO);
}
-
- //send task start event to TaskExecutor
- startTask(request.getTaskRequest(), resource);
- } else {
- // reject the exceeded requests
- response.addCancellationTask(request);
}
+ break;
}
- allocateEvent.getCallback().run(response.build());
+ case DEALLOCATE: {
+ NodeResourceDeallocateEvent deallocateEvent = TUtil.checkTypeAndGet(event, NodeResourceDeallocateEvent.class);
+ release(deallocateEvent.getResource());
- } else if (event instanceof NodeResourceDeallocateEvent) {
- NodeResourceDeallocateEvent deallocateEvent = (NodeResourceDeallocateEvent) event;
- release(deallocateEvent.getResource());
-
- // send current resource to ResourceTracker
- getDispatcher().getEventHandler().handle(
- new NodeStatusEvent(NodeStatusEvent.EventType.REPORT_RESOURCE));
+ if (deallocateEvent.getResourceType() == NodeResourceEvent.ResourceType.QUERY_MASTER) {
+ runningQueryMasters.decrementAndGet();
+ }
+ // send current resource to ResourceTracker
+ getDispatcher().getEventHandler().handle(
+ new NodeStatusEvent(NodeStatusEvent.EventType.REPORT_RESOURCE));
+ break;
+ }
}
}
- protected Dispatcher getDispatcher() {
+ public Dispatcher getDispatcher() {
return dispatcher;
}
- protected NodeResource getTotalResource() {
+ public NodeResource getTotalResource() {
return totalResource;
}
- protected NodeResource getAvailableResource() {
+ public NodeResource getAvailableResource() {
return availableResource;
}
+ public int getRunningQueryMasters() {
+ return runningQueryMasters.get();
+ }
+
private boolean allocate(NodeResource resource) {
- //TODO consider the jvm free memory
- if (NodeResources.fitsIn(resource, availableResource)) {
+
+ if (NodeResources.fitsIn(resource, availableResource) && checkFreeHeapMemory(resource)) {
NodeResources.subtractFrom(availableResource, resource);
return true;
}
return false;
}
- protected void startExecutionBlock(RunExecutionBlockRequestProto request) {
- taskEventHandler.handle(new ExecutionBlockStartEvent(request));
+ private boolean checkFreeHeapMemory(NodeResource resource) {
+ //TODO consider the jvm free memory
+ return true;
}
protected void startTask(TaskRequestProto request, NodeResource resource) {
- taskEventHandler.handle(new TaskStartEvent(request, resource));
+ workerContext.getTaskManager().getDispatcher().getEventHandler().handle(new TaskStartEvent(request, resource));
}
private void release(NodeResource resource) {
@@ -130,17 +155,19 @@ public class NodeResourceManager extends AbstractService implements EventHandler
}
private NodeResource createWorkerResource(TajoConf conf) {
- int memoryMb;
- if (conf.get(CommonTestingUtil.TAJO_TEST_KEY, "FALSE").equalsIgnoreCase("TRUE")) {
- memoryMb = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB);
- } else {
- memoryMb = Math.min((int) (Runtime.getRuntime().maxMemory() / StorageUnit.MB),
- conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB));
+ int memoryMb = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB);
+ if (!enableTest) {
+ // Set memory resource to max heap
+ int maxHeap = (int) (Runtime.getRuntime().maxMemory() / StorageUnit.MB);
+ if(maxHeap > memoryMb) {
+ memoryMb = maxHeap;
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB, memoryMb);
+ }
}
int vCores = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES);
- int disks = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS_NUM);
+ int disks = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS);
int dataNodeStorageSize = DiskUtil.getDataNodeStorageSize();
if (conf.getBoolVar(TajoConf.ConfVars.WORKER_RESOURCE_DFS_DIR_AWARE) && dataNodeStorageSize > 0) {
@@ -150,4 +177,23 @@ public class NodeResourceManager extends AbstractService implements EventHandler
int diskParallels = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM);
return NodeResource.createResource(memoryMb, disks * diskParallels, vCores);
}
+
+ private void validateConf(TajoConf conf) {
+ // validate node memory allocation setting
+ int minMem = conf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY);
+ int minQMMem = conf.getIntVar(TajoConf.ConfVars.QUERYMASTER_MINIMUM_MEMORY);
+ int maxMem = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB);
+
+ if (minMem <= 0 || minQMMem <= 0 || minMem + minQMMem > maxMem) {
+ throw new RuntimeException("Invalid resource worker memory"
+ + " allocation configuration"
+ + ", " + TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY.varname
+ + "=" + minMem
+ + ", " + TajoConf.ConfVars.QUERYMASTER_MINIMUM_MEMORY.varname
+ + "=" + minQMMem
+ + ", " + TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname
+ + "=" + maxMem + ", min and max should be greater than 0"
+ + ", max should be no smaller than min.");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java
index d13cd50..5d91cc6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java
@@ -27,11 +27,15 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.ipc.TajoResourceTrackerProtocol;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.resource.NodeResource;
-import org.apache.tajo.rpc.*;
+import org.apache.tajo.resource.DefaultResourceCalculator;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.rpc.AsyncRpcClient;
+import org.apache.tajo.rpc.CallFuture;
+import org.apache.tajo.rpc.RpcClientManager;
+import org.apache.tajo.rpc.RpcConstants;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.service.ServiceTrackerFactory;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.event.NodeStatusEvent;
import java.net.ConnectException;
@@ -42,8 +46,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.*;
+import static org.apache.tajo.ResourceProtos.*;
/**
* It periodically sends heartbeat to {@link org.apache.tajo.master.rm.TajoResourceTracker} via asynchronous rpc.
@@ -55,39 +58,42 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
private TajoConf tajoConf;
private StatusUpdaterThread updaterThread;
private volatile boolean isStopped;
- private volatile long heartBeatInterval;
+ private int heartBeatInterval;
+ private int nextHeartBeatInterval;
private BlockingQueue<NodeStatusEvent> heartBeatRequestQueue;
private final TajoWorker.WorkerContext workerContext;
- private final NodeResourceManager nodeResourceManager;
private AsyncRpcClient rmClient;
private ServiceTracker serviceTracker;
- private TajoResourceTrackerProtocolService.Interface resourceTracker;
- private int queueingLimit;
+ private TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interface resourceTracker;
+ private int queueingThreshold;
- public NodeStatusUpdater(TajoWorker.WorkerContext workerContext, NodeResourceManager resourceManager) {
+ public NodeStatusUpdater(TajoWorker.WorkerContext workerContext) {
super(NodeStatusUpdater.class.getSimpleName());
this.workerContext = workerContext;
- this.nodeResourceManager = resourceManager;
}
@Override
public void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- this.tajoConf = (TajoConf) conf;
+
+ this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
this.heartBeatRequestQueue = Queues.newLinkedBlockingQueue();
this.serviceTracker = ServiceTrackerFactory.get(tajoConf);
- this.nodeResourceManager.getDispatcher().register(NodeStatusEvent.EventType.class, this);
- this.heartBeatInterval = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_INTERVAL);
+ this.workerContext.getNodeResourceManager().getDispatcher().register(NodeStatusEvent.EventType.class, this);
+ this.heartBeatInterval = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL);
this.updaterThread = new StatusUpdaterThread();
+ this.updaterThread.setName("NodeStatusUpdater");
super.serviceInit(conf);
}
@Override
public void serviceStart() throws Exception {
// if resource changed over than 50%, send reports
- this.queueingLimit = nodeResourceManager.getTotalResource().getVirtualCores() / 2;
+ DefaultResourceCalculator calculator = new DefaultResourceCalculator();
+ int maxContainer = calculator.computeAvailableContainers(workerContext.getNodeResourceManager().getTotalResource(),
+ NodeResources.createResource(tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY), 1));
+
+ this.queueingThreshold = Math.max((int) Math.floor(maxContainer * 0.5), 1);
+ LOG.info("Queueing threshold:" + queueingThreshold);
updaterThread.start();
super.serviceStart();
@@ -97,10 +103,8 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
@Override
public void serviceStop() throws Exception {
this.isStopped = true;
-
synchronized (updaterThread) {
updaterThread.interrupt();
- updaterThread.join();
}
super.serviceStop();
LOG.info("NodeStatusUpdater stopped.");
@@ -115,35 +119,30 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
return heartBeatRequestQueue.size();
}
- public int getQueueingLimit() {
- return queueingLimit;
+ public int getQueueingThreshold() {
+ return queueingThreshold;
}
- private NodeHeartbeatRequestProto createResourceReport(NodeResource resource) {
- NodeHeartbeatRequestProto.Builder requestProto = NodeHeartbeatRequestProto.newBuilder();
- requestProto.setAvailableResource(resource.getProto());
+ private NodeHeartbeatRequest.Builder createResourceReport() {
+ NodeHeartbeatRequest.Builder requestProto = NodeHeartbeatRequest.newBuilder();
requestProto.setWorkerId(workerContext.getConnectionInfo().getId());
- return requestProto.build();
- }
+ requestProto.setAvailableResource(workerContext.getNodeResourceManager().getAvailableResource().getProto());
+ requestProto.setRunningTasks(workerContext.getTaskManager().getRunningTasks());
+ requestProto.setRunningQueryMasters(workerContext.getNodeResourceManager().getRunningQueryMasters());
- private NodeHeartbeatRequestProto createHeartBeatReport() {
- NodeHeartbeatRequestProto.Builder requestProto = NodeHeartbeatRequestProto.newBuilder();
- requestProto.setWorkerId(workerContext.getConnectionInfo().getId());
- return requestProto.build();
+ return requestProto;
}
- private NodeHeartbeatRequestProto createNodeStatusReport() {
- NodeHeartbeatRequestProto.Builder requestProto = NodeHeartbeatRequestProto.newBuilder();
- requestProto.setTotalResource(nodeResourceManager.getTotalResource().getProto());
- requestProto.setAvailableResource(nodeResourceManager.getAvailableResource().getProto());
- requestProto.setWorkerId(workerContext.getConnectionInfo().getId());
+ private NodeHeartbeatRequest.Builder createNodeStatusReport() {
+ NodeHeartbeatRequest.Builder requestProto = createResourceReport();
+ requestProto.setTotalResource(workerContext.getNodeResourceManager().getTotalResource().getProto());
requestProto.setConnectionInfo(workerContext.getConnectionInfo().getProto());
//TODO set node status to requestProto.setStatus()
- return requestProto.build();
+ return requestProto;
}
- protected TajoResourceTrackerProtocolService.Interface newStub()
+ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interface newStub()
throws NoSuchMethodException, ConnectException, ClassNotFoundException {
RpcClientManager.cleanup(rmClient);
@@ -154,15 +153,15 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
return rmClient.getStub();
}
- protected NodeHeartbeatResponseProto sendHeartbeat(NodeHeartbeatRequestProto requestProto)
+ protected NodeHeartbeatResponse sendHeartbeat(NodeHeartbeatRequest requestProto)
throws NoSuchMethodException, ClassNotFoundException, ConnectException, ExecutionException {
if (resourceTracker == null) {
resourceTracker = newStub();
}
- NodeHeartbeatResponseProto response = null;
+ NodeHeartbeatResponse response = null;
try {
- CallFuture<NodeHeartbeatResponseProto> callBack = new CallFuture<NodeHeartbeatResponseProto>();
+ CallFuture<NodeHeartbeatResponse> callBack = new CallFuture<NodeHeartbeatResponse>();
resourceTracker.nodeHeartbeat(callBack.getController(), requestProto, callBack);
response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
@@ -190,7 +189,6 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
long deadline = System.nanoTime() + unit.toNanos(timeout);
int added = 0;
while (added < numElements) {
- added += heartBeatRequestQueue.drainTo(buffer, numElements - added);
if (added < numElements) { // not enough elements immediately available; will have to wait
NodeStatusEvent e = heartBeatRequestQueue.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS);
if (e == null) {
@@ -200,6 +198,7 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
added++;
if (e.getType() == NodeStatusEvent.EventType.FLUSH_REPORTS) {
+ added += heartBeatRequestQueue.drainTo(buffer, numElements - added);
break;
}
}
@@ -210,37 +209,39 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
/* Node sends a heartbeats with its resource and status periodically to master. */
@Override
public void run() {
- NodeHeartbeatResponseProto lastResponse = null;
+ NodeHeartbeatResponse lastResponse = null;
while (!isStopped && !Thread.interrupted()) {
try {
if (lastResponse != null) {
if (lastResponse.getCommand() == ResponseCommand.NORMAL) {
List<NodeStatusEvent> events = Lists.newArrayList();
+
+ if(lastResponse.hasHeartBeatInterval()) {
+ nextHeartBeatInterval = lastResponse.getHeartBeatInterval();
+ } else {
+ nextHeartBeatInterval = heartBeatInterval;
+ }
+
try {
/* batch update to ResourceTracker */
- drain(events, Math.max(queueingLimit, 1), heartBeatInterval, TimeUnit.MILLISECONDS);
+ drain(events, queueingThreshold, nextHeartBeatInterval, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
break;
}
- if (!events.isEmpty()) {
- // send current available resource;
- lastResponse = sendHeartbeat(createResourceReport(nodeResourceManager.getAvailableResource()));
- } else {
- // send ping;
- lastResponse = sendHeartbeat(createHeartBeatReport());
- }
+ // send current available resource;
+ lastResponse = sendHeartbeat(createResourceReport().build());
} else if (lastResponse.getCommand() == ResponseCommand.MEMBERSHIP) {
// Membership changed
- lastResponse = sendHeartbeat(createNodeStatusReport());
+ lastResponse = sendHeartbeat(createNodeStatusReport().build());
} else if (lastResponse.getCommand() == ResponseCommand.ABORT_QUERY) {
//TODO abort failure queries
}
} else {
// Node registration on startup
- lastResponse = sendHeartbeat(createNodeStatusReport());
+ lastResponse = sendHeartbeat(createNodeStatusReport().build());
}
} catch (NoSuchMethodException nsme) {
LOG.fatal(nsme.getMessage(), nsme);
@@ -249,15 +250,10 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler<N
LOG.fatal(cnfe.getMessage(), cnfe);
Runtime.getRuntime().halt(-1);
} catch (Exception e) {
- LOG.error(e.getMessage(), e);
- if (!isStopped) {
- synchronized (updaterThread) {
- try {
- updaterThread.wait(heartBeatInterval);
- } catch (InterruptedException ie) {
- // Do Nothing
- }
- }
+ if (isStopped) {
+ break;
+ } else {
+ LOG.error(e.getMessage(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
deleted file mode 100644
index b713e70..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.master.container.TajoContainerId;
-
-public interface ResourceAllocator {
- public void allocateTaskWorker();
- public TajoContainerId makeContainerId(ContainerProtocol.TajoContainerIdProto containerId);
- public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext,
- int numTasks, int memoryMBPerTask);
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
deleted file mode 100644
index 05dd1a9..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
+++ /dev/null
@@ -1,415 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.*;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.*;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.container.TajoContainer;
-import org.apache.tajo.master.container.TajoContainerId;
-import org.apache.tajo.master.event.ContainerAllocationEvent;
-import org.apache.tajo.master.event.ContainerAllocatorEventType;
-import org.apache.tajo.master.event.StageContainerAllocationEvent;
-import org.apache.tajo.master.rm.TajoWorkerContainer;
-import org.apache.tajo.master.rm.TajoWorkerContainerId;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
-import org.apache.tajo.querymaster.QueryMasterTask;
-import org.apache.tajo.querymaster.Stage;
-import org.apache.tajo.querymaster.StageState;
-import org.apache.tajo.rpc.CallFuture;
-import org.apache.tajo.rpc.NettyClientBase;
-import org.apache.tajo.rpc.NullCallback;
-import org.apache.tajo.rpc.RpcClientManager;
-import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
-import org.apache.tajo.service.ServiceTracker;
-import org.apache.tajo.util.ApplicationIdUtils;
-
-import java.net.InetSocketAddress;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class TajoResourceAllocator extends AbstractResourceAllocator {
- private static final Log LOG = LogFactory.getLog(TajoResourceAllocator.class);
-
- private TajoConf tajoConf;
- private QueryMasterTask.QueryMasterTaskContext queryTaskContext;
- private final ExecutorService allocationExecutor;
- private final Deallocator deallocator;
-
- private AtomicBoolean stopped = new AtomicBoolean(false);
-
- public TajoResourceAllocator(QueryMasterTask.QueryMasterTaskContext queryTaskContext) {
- this.queryTaskContext = queryTaskContext;
- allocationExecutor = Executors.newFixedThreadPool(
- queryTaskContext.getConf().getIntVar(TajoConf.ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM));
- deallocator = new Deallocator();
- }
-
- @Override
- public TajoContainerId makeContainerId(ContainerProtocol.TajoContainerIdProto containerIdProto) {
- TajoWorkerContainerId containerId = new TajoWorkerContainerId();
- ApplicationAttemptId appAttemptId = new ApplicationAttemptIdPBImpl(containerIdProto.getAppAttemptId());
- containerId.setApplicationAttemptId(appAttemptId);
- containerId.setId(containerIdProto.getId());
- return containerId;
- }
-
- @Override
- public void allocateTaskWorker() {
- }
-
- @Override
- public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext,
- int numTasks,
- int memoryMBPerTask) {
- //TODO consider disk slot
-
- ClusterResourceSummary clusterResource = workerContext.getClusterResource();
- int clusterSlots = clusterResource == null ? 0 : clusterResource.getTotalMemoryMB() / memoryMBPerTask;
- clusterSlots = Math.max(1, clusterSlots - 1); // reserve query master slot
- LOG.info("CalculateNumberRequestContainer - Number of Tasks=" + numTasks +
- ", Number of Cluster Slots=" + clusterSlots);
- return Math.min(numTasks, clusterSlots);
- }
-
- @Override
- public void init(Configuration conf) {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("conf should be a TajoConf type.");
- }
- tajoConf = (TajoConf)conf;
-
- queryTaskContext.getDispatcher().register(TaskRunnerGroupEvent.EventType.class, new TajoTaskRunnerLauncher());
-
- queryTaskContext.getDispatcher().register(ContainerAllocatorEventType.class, new TajoWorkerAllocationHandler());
-
- deallocator.start();
-
- super.init(conf);
- }
-
- @Override
- public synchronized void stop() {
- if (stopped.compareAndSet(false, true)) {
- return;
- }
-
- allocationExecutor.shutdownNow();
- deallocator.shutdown();
-
- Map<TajoContainerId, ContainerProxy> containers = queryTaskContext.getResourceAllocator()
- .getContainers();
- List<ContainerProxy> list = new ArrayList<ContainerProxy>(containers.values());
- for(ContainerProxy eachProxy: list) {
- try {
- eachProxy.stopContainer();
- } catch (Throwable e) {
- LOG.warn(e.getMessage(), e);
- }
- }
-
- workerInfoMap.clear();
- super.stop();
- }
-
- @Override
- public void start() {
- super.start();
- }
-
- class TajoTaskRunnerLauncher implements TaskRunnerLauncher {
- @Override
- public void handle(TaskRunnerGroupEvent event) {
- if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_LAUNCH) {
- if (!(event instanceof LaunchTaskRunnersEvent)) {
- throw new IllegalArgumentException("event should be a LaunchTaskRunnersEvent type.");
- }
- LaunchTaskRunnersEvent launchEvent = (LaunchTaskRunnersEvent) event;
- launchTaskRunners(launchEvent);
- } else if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_CLEANUP) {
- stopContainers(event.getContainers());
- stopExecutionBlock(event.getExecutionBlockId(), event.getContainers());
- }
- }
- }
-
- private void launchTaskRunners(LaunchTaskRunnersEvent event) {
- // Query in standby mode doesn't need launch Worker.
- // But, Assign ExecutionBlock to assigned tajo worker
- for(TajoContainer eachContainer: event.getContainers()) {
- TajoContainerProxy containerProxy = new TajoContainerProxy(queryTaskContext, tajoConf,
- eachContainer, event.getQueryContext(), event.getExecutionBlockId(), event.getPlanJson());
- allocationExecutor.submit(new LaunchRunner(eachContainer.getId(), containerProxy));
- }
- }
-
- public void stopExecutionBlock(final ExecutionBlockId executionBlockId,
- Collection<TajoContainer> containers) {
- Set<NodeId> workers = Sets.newHashSet();
- for (TajoContainer container : containers){
- workers.add(container.getNodeId());
- }
-
- for (final NodeId worker : workers) {
- allocationExecutor.submit(new Runnable() {
- @Override
- public void run() {
- stopExecutionBlock(executionBlockId, worker);
- }
- });
- }
- }
-
- private void stopExecutionBlock(ExecutionBlockId executionBlockId, NodeId worker) {
- NettyClientBase tajoWorkerRpc = null;
- try {
- InetSocketAddress addr = new InetSocketAddress(worker.getHost(), worker.getPort());
- tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true);
- TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
-
- tajoWorkerRpcClient.stopExecutionBlock(null, executionBlockId.getProto(),
- NullCallback.get(PrimitiveProtos.BoolProto.class));
- } catch (Throwable e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- protected static class LaunchRunner implements Runnable {
- private final ContainerProxy proxy;
- private final TajoContainerId id;
- public LaunchRunner(TajoContainerId id, ContainerProxy proxy) {
- this.proxy = proxy;
- this.id = id;
- }
- @Override
- public void run() {
- proxy.launch(null);
- if (LOG.isDebugEnabled()) {
- LOG.debug("ContainerProxy started:" + id);
- }
- }
- }
-
- private void stopContainers(Collection<TajoContainer> containers) {
- deallocator.submit(Iterables.transform(containers, new Function<TajoContainer, TajoContainerId>() {
- public TajoContainerId apply(TajoContainer input) { return input.getId(); }
- }));
- }
-
- private static final TajoContainerId FIN = new TajoWorkerContainerId();
-
- private class Deallocator extends Thread {
-
- private final BlockingDeque<TajoContainerId> queue = new LinkedBlockingDeque<TajoContainerId>();
-
- public Deallocator() {
- setName("Deallocator");
- setDaemon(true);
- }
-
- private void submit(Iterable<TajoContainerId> container) {
- queue.addAll(Lists.newArrayList(container));
- }
-
- private void shutdown() {
- queue.add(FIN);
- }
-
- @Override
- public void run() {
- final AbstractResourceAllocator allocator = queryTaskContext.getResourceAllocator();
- while (!stopped.get() || !queue.isEmpty()) {
- TajoContainerId containerId;
- try {
- containerId = queue.take();
- } catch (InterruptedException e) {
- continue;
- }
- if (containerId == FIN) {
- break;
- }
- ContainerProxy proxy = allocator.getContainer(containerId);
- if (proxy == null) {
- continue;
- }
- try {
- LOG.info("Stopping ContainerProxy: " + proxy.getContainerId() + "," + proxy.getBlockId());
- proxy.stopContainer();
- } catch (Exception e) {
- LOG.warn("Failed to stop container " + proxy.getContainerId() + "," + proxy.getBlockId(), e);
- }
- }
- LOG.info("Deallocator exiting");
- }
- }
-
- class TajoWorkerAllocationHandler implements EventHandler<ContainerAllocationEvent> {
- @Override
- public void handle(ContainerAllocationEvent event) {
- allocationExecutor.submit(new TajoWorkerAllocationThread(event));
- }
- }
-
- class TajoWorkerAllocationThread extends Thread {
- ContainerAllocationEvent event;
- TajoWorkerAllocationThread(ContainerAllocationEvent event) {
- this.event = event;
- }
-
- @Override
- public void run() {
- LOG.info("Start TajoWorkerAllocationThread");
- CallFuture<WorkerResourceAllocationResponse> callBack =
- new CallFuture<WorkerResourceAllocationResponse>();
-
- //TODO consider task's resource usage pattern
- int requiredMemoryMB = tajoConf.getIntVar(TajoConf.ConfVars.TASK_DEFAULT_MEMORY);
- float requiredDiskSlots = tajoConf.getFloatVar(TajoConf.ConfVars.TASK_DEFAULT_DISK);
-
- WorkerResourceAllocationRequest request = WorkerResourceAllocationRequest.newBuilder()
- .setMinMemoryMBPerContainer(requiredMemoryMB)
- .setMaxMemoryMBPerContainer(requiredMemoryMB)
- .setNumContainers(event.getRequiredNum())
- .setResourceRequestPriority(!event.isLeafQuery() ?
- ResourceRequestPriority.MEMORY : ResourceRequestPriority.DISK)
- .setMinDiskSlotPerContainer(requiredDiskSlots)
- .setMaxDiskSlotPerContainer(requiredDiskSlots)
- .setQueryId(event.getExecutionBlockId().getQueryId().getProto())
- .build();
-
-
- NettyClientBase tmClient = null;
- try {
- ServiceTracker serviceTracker = queryTaskContext.getQueryMasterContext().getWorkerContext().getServiceTracker();
- tmClient = RpcClientManager.getInstance().
- getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true);
- QueryCoordinatorProtocolService masterClientService = tmClient.getStub();
- masterClientService.allocateWorkerResources(callBack.getController(), request, callBack);
- } catch (Throwable e) {
- LOG.error(e.getMessage(), e);
- }
-
- WorkerResourceAllocationResponse response = null;
- while(!stopped.get()) {
- try {
- response = callBack.get(3, TimeUnit.SECONDS);
- break;
- } catch (InterruptedException e) {
- if(stopped.get()) {
- return;
- }
- } catch (TimeoutException e) {
- LOG.info("No available worker resource for " + event.getExecutionBlockId());
- continue;
- } catch (ExecutionException e) {
- LOG.error(e.getMessage(), e);
- break;
- }
- }
-
- int numAllocatedContainers = 0;
-
- if(response != null) {
- List<WorkerAllocatedResource> allocatedResources = response.getWorkerAllocatedResourceList();
- ExecutionBlockId executionBlockId = event.getExecutionBlockId();
-
- List<TajoContainer> containers = new ArrayList<TajoContainer>();
- for(WorkerAllocatedResource eachAllocatedResource: allocatedResources) {
- TajoWorkerContainer container = new TajoWorkerContainer();
- NodeId nodeId = NodeId.newInstance(eachAllocatedResource.getConnectionInfo().getHost(),
- eachAllocatedResource.getConnectionInfo().getPeerRpcPort());
-
- TajoWorkerContainerId containerId = new TajoWorkerContainerId();
-
- containerId.setApplicationAttemptId(
- ApplicationIdUtils.createApplicationAttemptId(executionBlockId.getQueryId(),
- eachAllocatedResource.getContainerId().getAppAttemptId().getAttemptId()));
- containerId.setId(eachAllocatedResource.getContainerId().getId());
-
- container.setId(containerId);
- container.setNodeId(nodeId);
-
-
- WorkerResource workerResource = new WorkerResource();
- workerResource.setMemoryMB(eachAllocatedResource.getAllocatedMemoryMB());
- workerResource.setDiskSlots(eachAllocatedResource.getAllocatedDiskSlots());
-
- Worker worker = new Worker(null, workerResource,
- new WorkerConnectionInfo(eachAllocatedResource.getConnectionInfo()));
- container.setWorkerResource(worker);
- addWorkerConnectionInfo(worker.getConnectionInfo());
- containers.add(container);
- }
-
- StageState state = queryTaskContext.getStage(executionBlockId).getSynchronizedState();
- if (!Stage.isRunningState(state)) {
- List<TajoContainerId> containerIds = new ArrayList<TajoContainerId>();
- for(TajoContainer eachContainer: containers) {
- containerIds.add(eachContainer.getId());
- }
- try {
- TajoContainerProxy.releaseWorkerResource(queryTaskContext, executionBlockId, containerIds);
- } catch (Throwable e) {
- deallocator.submit(containerIds);
- LOG.error(e.getMessage(), e);
- }
- return;
- }
-
- if (allocatedResources.size() > 0) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("StageContainerAllocationEvent fire:" + executionBlockId);
- }
- queryTaskContext.getEventHandler().handle(new StageContainerAllocationEvent(executionBlockId, containers));
- }
- numAllocatedContainers += allocatedResources.size();
-
- }
- if(event.getRequiredNum() > numAllocatedContainers) {
- ContainerAllocationEvent shortRequestEvent = new ContainerAllocationEvent(
- event.getType(), event.getExecutionBlockId(), event.getPriority(),
- event.getResource(),
- event.getRequiredNum() - numAllocatedContainers,
- event.isLeafQuery(), event.getProgress()
- );
- queryTaskContext.getEventHandler().handle(shortRequestEvent);
-
- }
- LOG.info("Stop TajoWorkerAllocationThread");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
index 66c8e4a..a95698f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
@@ -42,9 +42,7 @@ import org.apache.tajo.rpc.RpcConstants;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.service.ServiceTrackerFactory;
import org.apache.tajo.service.TajoMasterInfo;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.ClusterResourceSummary;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.rm.TajoWorkerResourceManager;
import org.apache.tajo.pullserver.TajoPullServerService;
import org.apache.tajo.querymaster.QueryMaster;
import org.apache.tajo.querymaster.QueryMasterManagerService;
@@ -70,7 +68,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.tajo.conf.TajoConf.ConfVars;
@@ -97,19 +94,19 @@ public class TajoWorker extends CompositeService {
private WorkerContext workerContext;
- private TaskRunnerManager taskRunnerManager;
+ private TaskManager taskManager;
+
+ private TaskExecutor taskExecutor;
private TajoPullServerService pullService;
private ServiceTracker serviceTracker;
- private WorkerHeartbeatService workerHeartbeatThread;
-
- private AtomicBoolean stopped = new AtomicBoolean(false);
+ private NodeResourceManager nodeResourceManager;
- private AtomicInteger numClusterNodes = new AtomicInteger();
+ private NodeStatusUpdater nodeStatusUpdater;
- private ClusterResourceSummary clusterResource;
+ private AtomicBoolean stopped = new AtomicBoolean(false);
private WorkerConnectionInfo connectionInfo;
@@ -147,12 +144,9 @@ public class TajoWorker extends CompositeService {
@Override
public void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("conf should be a TajoConf type.");
- }
Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));
- this.systemConf = (TajoConf)conf;
+ this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
RackResolver.init(systemConf);
RpcClientManager rpcManager = RpcClientManager.getInstance();
@@ -165,23 +159,11 @@ public class TajoWorker extends CompositeService {
this.workerContext = new TajoWorkerContext();
this.lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);
- String resourceManagerClassName = systemConf.getVar(ConfVars.RESOURCE_MANAGER_CLASS);
-
- boolean randomPort = true;
- if(resourceManagerClassName.indexOf(TajoWorkerResourceManager.class.getName()) >= 0) {
- randomPort = false;
- }
int clientPort = systemConf.getSocketAddrVar(ConfVars.WORKER_CLIENT_RPC_ADDRESS).getPort();
int peerRpcPort = systemConf.getSocketAddrVar(ConfVars.WORKER_PEER_RPC_ADDRESS).getPort();
int qmManagerPort = systemConf.getSocketAddrVar(ConfVars.WORKER_QM_RPC_ADDRESS).getPort();
- if(randomPort) {
- clientPort = 0;
- peerRpcPort = 0;
- qmManagerPort = 0;
- systemConf.setIntVar(ConfVars.PULLSERVER_PORT, 0);
- }
this.dispatcher = new AsyncDispatcher();
addIfService(dispatcher);
@@ -196,12 +178,19 @@ public class TajoWorker extends CompositeService {
queryMasterManagerService = new QueryMasterManagerService(workerContext, qmManagerPort);
addIfService(queryMasterManagerService);
- // taskrunner worker
- taskRunnerManager = new TaskRunnerManager(workerContext, dispatcher);
- addService(taskRunnerManager);
+ this.taskManager = new TaskManager(dispatcher, workerContext);
+ addService(taskManager);
- workerHeartbeatThread = new WorkerHeartbeatService(workerContext);
- addIfService(workerHeartbeatThread);
+ this.taskExecutor = new TaskExecutor(workerContext);
+ addService(taskExecutor);
+
+ AsyncDispatcher rmDispatcher = new AsyncDispatcher();
+ addService(rmDispatcher);
+ this.nodeResourceManager = new NodeResourceManager(rmDispatcher, workerContext);
+ addService(nodeResourceManager);
+
+ this.nodeStatusUpdater = new NodeStatusUpdater(workerContext);
+ addService(nodeStatusUpdater);
int httpPort = 0;
if(!TajoPullServerService.isStandalone()) {
@@ -268,8 +257,8 @@ public class TajoWorker extends CompositeService {
workerSystemMetrics.register("task", "runningTasks", new Gauge<Integer>() {
@Override
public Integer getValue() {
- if(taskRunnerManager != null) {
- return taskRunnerManager.getNumTasks();
+ if(taskExecutor != null) {
+ return taskExecutor.getRunningTasks();
} else {
return 0;
}
@@ -394,7 +383,11 @@ public class TajoWorker extends CompositeService {
QueryMasterManagerService getQueryMasterManagerService();
- TaskRunnerManager getTaskRunnerManager();
+ TaskManager getTaskManager();
+
+ TaskExecutor getTaskExecuor();
+
+ NodeResourceManager getNodeResourceManager();
CatalogService getCatalog();
@@ -404,8 +397,6 @@ public class TajoWorker extends CompositeService {
LocalDirAllocator getLocalDirAllocator();
- ClusterResourceSummary getClusterResource();
-
TajoSystemMetrics getWorkerSystemMetrics();
HashShuffleAppenderManager getHashShuffleAppenderManager();
@@ -417,10 +408,6 @@ public class TajoWorker extends CompositeService {
void cleanup(String strPath);
void cleanupTemporalDirectories();
-
- void setClusterResource(ClusterResourceSummary clusterResource);
-
- void setNumClusterNodes(int numClusterNodes);
}
class TajoWorkerContext implements WorkerContext {
@@ -443,8 +430,19 @@ public class TajoWorker extends CompositeService {
return queryMasterManagerService;
}
- public TaskRunnerManager getTaskRunnerManager() {
- return taskRunnerManager;
+ @Override
+ public TaskManager getTaskManager(){
+ return taskManager;
+ }
+
+ @Override
+ public TaskExecutor getTaskExecuor() {
+ return taskExecutor;
+ }
+
+ @Override
+ public NodeResourceManager getNodeResourceManager() {
+ return nodeResourceManager;
}
public CatalogService getCatalog() {
@@ -507,22 +505,6 @@ public class TajoWorker extends CompositeService {
}
}
- public void setNumClusterNodes(int numClusterNodes) {
- TajoWorker.this.numClusterNodes.set(numClusterNodes);
- }
-
- public void setClusterResource(ClusterResourceSummary clusterResource) {
- synchronized (numClusterNodes) {
- TajoWorker.this.clusterResource = clusterResource;
- }
- }
-
- public ClusterResourceSummary getClusterResource() {
- synchronized (numClusterNodes) {
- return TajoWorker.this.clusterResource;
- }
- }
-
public TajoSystemMetrics getWorkerSystemMetrics() {
return workerSystemMetrics;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
index de8afe8..7752211 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
@@ -18,26 +18,27 @@
package org.apache.tajo.worker;
-import com.google.common.base.Preconditions;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
-import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
-import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.ResourceProtos.BatchAllocationRequest;
+import org.apache.tajo.ResourceProtos.BatchAllocationResponse;
+import org.apache.tajo.ResourceProtos.StopExecutionBlockRequest;
import org.apache.tajo.TajoIdProtos;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.rpc.AsyncRpcServer;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.util.NetUtils;
-import org.apache.tajo.worker.event.TaskRunnerStartEvent;
-import org.apache.tajo.worker.event.TaskRunnerStopEvent;
+import org.apache.tajo.util.TUtil;
+import org.apache.tajo.worker.event.ExecutionBlockStopEvent;
+import org.apache.tajo.worker.event.NodeResourceAllocateEvent;
+import org.apache.tajo.worker.event.QueryStopEvent;
import java.net.InetSocketAddress;
@@ -58,9 +59,9 @@ public class TajoWorkerManagerService extends CompositeService
}
@Override
- public void init(Configuration conf) {
- Preconditions.checkArgument(conf instanceof TajoConf);
- TajoConf tajoConf = (TajoConf) conf;
+ public void serviceInit(Configuration conf) throws Exception {
+
+ TajoConf tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
try {
// Setup RPC server
InetSocketAddress initIsa =
@@ -81,21 +82,16 @@ public class TajoWorkerManagerService extends CompositeService
// Get the master address
LOG.info("TajoWorkerManagerService is bind to " + bindAddr);
tajoConf.setVar(TajoConf.ConfVars.WORKER_PEER_RPC_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddr));
- super.init(tajoConf);
+ super.serviceInit(tajoConf);
}
@Override
- public void start() {
- super.start();
- }
-
- @Override
- public void stop() {
+ public void serviceStop() throws Exception {
if(rpcServer != null) {
rpcServer.shutdown();
}
LOG.info("TajoWorkerManagerService stopped");
- super.stop();
+ super.serviceStop();
}
public InetSocketAddress getBindAddr() {
@@ -110,29 +106,23 @@ public class TajoWorkerManagerService extends CompositeService
}
@Override
- public void startExecutionBlock(RpcController controller,
- TajoWorkerProtocol.RunExecutionBlockRequestProto request,
- RpcCallback<PrimitiveProtos.BoolProto> done) {
- workerContext.getWorkerSystemMetrics().counter("query", "executedExecutionBlocksNum").inc();
-
- try {
- workerContext.getTaskRunnerManager().getEventHandler().handle(new TaskRunnerStartEvent(request));
- done.run(TajoWorker.TRUE_PROTO);
- } catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- controller.setFailed(t.getMessage());
- done.run(TajoWorker.FALSE_PROTO);
- }
+ public void allocateTasks(RpcController controller,
+ BatchAllocationRequest request,
+ RpcCallback<BatchAllocationResponse> done) {
+ workerContext.getWorkerSystemMetrics().counter("query", "allocationRequestNum").inc();
+ workerContext.getNodeResourceManager().getDispatcher().
+ getEventHandler().handle(new NodeResourceAllocateEvent(request, done));
}
@Override
public void stopExecutionBlock(RpcController controller,
- TajoIdProtos.ExecutionBlockIdProto requestProto,
+ StopExecutionBlockRequest requestProto,
RpcCallback<PrimitiveProtos.BoolProto> done) {
try {
- workerContext.getTaskRunnerManager().getEventHandler().handle(new TaskRunnerStopEvent(
- new ExecutionBlockId(requestProto)
- ));
+
+ workerContext.getTaskManager().getDispatcher().getEventHandler().handle(
+ new ExecutionBlockStopEvent(requestProto.getExecutionBlockId(), requestProto.getCleanupList()));
+
done.run(TajoWorker.TRUE_PROTO);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -144,29 +134,18 @@ public class TajoWorkerManagerService extends CompositeService
@Override
public void killTaskAttempt(RpcController controller, TajoIdProtos.TaskAttemptIdProto request,
RpcCallback<PrimitiveProtos.BoolProto> done) {
- Task task = workerContext.getTaskRunnerManager().getTaskByTaskAttemptId(new TaskAttemptId(request));
+ //TODO change to async ?
+ Task task = workerContext.getTaskManager().getTaskByTaskAttemptId(new TaskAttemptId(request));
if(task != null) task.kill();
done.run(TajoWorker.TRUE_PROTO);
}
@Override
- public void cleanup(RpcController controller, TajoIdProtos.QueryIdProto request,
+ public void stopQuery(RpcController controller, TajoIdProtos.QueryIdProto request,
RpcCallback<PrimitiveProtos.BoolProto> done) {
- workerContext.cleanup(new QueryId(request).toString());
- done.run(TajoWorker.TRUE_PROTO);
- }
- @Override
- public void cleanupExecutionBlocks(RpcController controller,
- TajoWorkerProtocol.ExecutionBlockListProto ebIds,
- RpcCallback<PrimitiveProtos.BoolProto> done) {
- for (TajoIdProtos.ExecutionBlockIdProto executionBlockIdProto : ebIds.getExecutionBlockIdList()) {
- String inputDir = ExecutionBlockContext.getBaseInputDir(new ExecutionBlockId(executionBlockIdProto)).toString();
- workerContext.cleanup(inputDir);
- String outputDir = ExecutionBlockContext.getBaseOutputDir(new ExecutionBlockId(executionBlockIdProto)).toString();
- workerContext.cleanup(outputDir);
- }
+ workerContext.getTaskManager().getDispatcher().getEventHandler().handle(new QueryStopEvent(new QueryId(request)));
done.run(TajoWorker.TRUE_PROTO);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index c849940..66216ee 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -18,9 +18,10 @@
package org.apache.tajo.worker;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ResourceProtos.TaskStatusProto;
import java.io.IOException;
+import java.util.List;
public interface Task {
@@ -48,5 +49,9 @@ public interface Task {
ExecutionBlockContext getExecutionBlockContext();
- TajoWorkerProtocol.TaskStatusProto getReport();
+ TaskStatusProto getReport();
+
+ TaskHistory createTaskHistory();
+
+ List<Fetcher> getFetchers();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java
index 2576726..761bf52 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java
@@ -20,7 +20,6 @@ package org.apache.tajo.worker;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.TajoProtos;
/**
* The driver class for Tajo Task processing.
[08/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
deleted file mode 100644
index 90a4eb5..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
+++ /dev/null
@@ -1,605 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import com.google.protobuf.RpcCallback;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.*;
-import org.apache.tajo.master.QueryInProgress;
-import org.apache.tajo.master.TajoMaster;
-import org.apache.tajo.rpc.CancelableRpcCallback;
-import org.apache.tajo.rpc.RpcUtils;
-import org.apache.tajo.util.ApplicationIdUtils;
-import org.apache.tajo.util.BasicFuture;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-
-/**
- * It manages all resources of tajo workers.
- */
-public class TajoWorkerResourceManager extends CompositeService implements WorkerResourceManager {
- /** class logger */
- private static final Log LOG = LogFactory.getLog(TajoWorkerResourceManager.class);
-
- static AtomicInteger containerIdSeq = new AtomicInteger(0);
-
- private TajoMaster.MasterContext masterContext;
-
- private TajoRMContext rmContext;
-
- private String queryIdSeed;
-
- private WorkerResourceAllocationThread workerResourceAllocator;
-
- /**
- * Worker Liveliness monitor
- */
- private WorkerLivelinessMonitor workerLivelinessMonitor;
-
- private final BlockingQueue<WorkerResourceRequest> requestQueue =
- new LinkedBlockingDeque<WorkerResourceRequest>();
- private final RpcUtils.Scrutineer<BasicFuture<ClusterResourceSummary>> summaryRequest =
- new RpcUtils.Scrutineer<BasicFuture<ClusterResourceSummary>>();
-
- private AtomicBoolean stopped = new AtomicBoolean(false);
-
- private TajoConf systemConf;
-
- private ConcurrentMap<ContainerProtocol.TajoContainerIdProto, AllocatedWorkerResource> allocatedResourceMap = Maps
- .newConcurrentMap();
-
- /** It receives status messages from workers and their resources. */
- private TajoResourceTracker resourceTracker;
-
- public TajoWorkerResourceManager(TajoMaster.MasterContext masterContext) {
- super(TajoWorkerResourceManager.class.getSimpleName());
- this.masterContext = masterContext;
- }
-
- public TajoWorkerResourceManager(TajoConf systemConf) {
- super(TajoWorkerResourceManager.class.getSimpleName());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- Preconditions.checkArgument(conf instanceof TajoConf);
- this.systemConf = (TajoConf) conf;
-
- AsyncDispatcher dispatcher = new AsyncDispatcher();
- addIfService(dispatcher);
-
- rmContext = new TajoRMContext(dispatcher);
-
- this.queryIdSeed = String.valueOf(System.currentTimeMillis());
-
- workerResourceAllocator = new WorkerResourceAllocationThread();
- workerResourceAllocator.start();
-
- this.workerLivelinessMonitor = new WorkerLivelinessMonitor(this.rmContext.getDispatcher());
- addIfService(this.workerLivelinessMonitor);
-
- // Register event handler for Workers
- rmContext.getDispatcher().register(WorkerEventType.class, new WorkerEventDispatcher(rmContext));
-
- resourceTracker = new TajoResourceTracker(this, workerLivelinessMonitor);
- addIfService(resourceTracker);
-
- super.serviceInit(systemConf);
- }
-
- @InterfaceAudience.Private
- public static final class WorkerEventDispatcher implements EventHandler<WorkerEvent> {
-
- private final TajoRMContext rmContext;
-
- public WorkerEventDispatcher(TajoRMContext rmContext) {
- this.rmContext = rmContext;
- }
-
- @Override
- public void handle(WorkerEvent event) {
- int workerId = event.getWorkerId();
- Worker node = this.rmContext.getWorkers().get(workerId);
- if (node != null) {
- try {
- node.handle(event);
- } catch (Throwable t) {
- LOG.error("Error in handling event type " + event.getType() + " for node " + workerId, t);
- }
- }
- }
- }
-
- @Override
- public Map<Integer, Worker> getWorkers() {
- return ImmutableMap.copyOf(rmContext.getWorkers());
- }
-
- @Override
- public Map<Integer, Worker> getInactiveWorkers() {
- return ImmutableMap.copyOf(rmContext.getInactiveWorkers());
- }
-
- public Collection<Integer> getQueryMasters() {
- return Collections.unmodifiableSet(rmContext.getQueryMasterWorker());
- }
-
- @Override
- public void serviceStop() throws Exception {
- if(stopped.get()) {
- return;
- }
- stopped.set(true);
- if(workerResourceAllocator != null) {
- workerResourceAllocator.interrupt();
- }
-
- super.serviceStop();
- }
-
- /**
- *
- * @return The prefix of queryId. It is generated when a TajoMaster starts up.
- */
- @Override
- public String getSeedQueryId() throws IOException {
- return queryIdSeed;
- }
-
- @VisibleForTesting
- TajoResourceTracker getResourceTracker() {
- return resourceTracker;
- }
-
- private WorkerResourceAllocationRequest createQMResourceRequest(QueryId queryId) {
- float queryMasterDefaultDiskSlot = masterContext.getConf().getFloatVar(
- TajoConf.ConfVars.TAJO_QUERYMASTER_DISK_SLOT);
- int queryMasterDefaultMemoryMB = masterContext.getConf().getIntVar(TajoConf.ConfVars.TAJO_QUERYMASTER_MEMORY_MB);
-
- WorkerResourceAllocationRequest.Builder builder = WorkerResourceAllocationRequest.newBuilder();
- builder.setQueryId(queryId.getProto());
- builder.setMaxMemoryMBPerContainer(queryMasterDefaultMemoryMB);
- builder.setMinMemoryMBPerContainer(queryMasterDefaultMemoryMB);
- builder.setMaxDiskSlotPerContainer(queryMasterDefaultDiskSlot);
- builder.setMinDiskSlotPerContainer(queryMasterDefaultDiskSlot);
- builder.setResourceRequestPriority(ResourceRequestPriority.MEMORY);
- builder.setNumContainers(1);
- return builder.build();
- }
-
- @Override
- public WorkerAllocatedResource allocateQueryMaster(QueryInProgress queryInProgress) {
-
- // 3 seconds, by default
- long timeout = masterContext.getConf().getTimeVar(
- TajoConf.ConfVars.TAJO_QUERYMASTER_ALLOCATION_TIMEOUT, TimeUnit.MILLISECONDS);
-
- // Create a resource request for a query master
- WorkerResourceAllocationRequest qmResourceRequest = createQMResourceRequest(queryInProgress.getQueryId());
-
- // call future for async call
- final CancelableRpcCallback<WorkerResourceAllocationResponse> callFuture =
- new CancelableRpcCallback<WorkerResourceAllocationResponse>() {
- @Override
- protected void cancel(WorkerResourceAllocationResponse canceled) {
- if (canceled != null && !canceled.getWorkerAllocatedResourceList().isEmpty()) {
- LOG.info("Canceling resources allocated");
- WorkerAllocatedResource resource = canceled.getWorkerAllocatedResource(0);
- releaseWorkerResource(resource.getContainerId());
- }
- }
- };
- allocateWorkerResources(qmResourceRequest, callFuture);
-
- WorkerResourceAllocationResponse response = null;
- try {
- response = callFuture.get(timeout, TimeUnit.MILLISECONDS);
- } catch (Throwable t) {
- response = callFuture.cancel(); // try cancel
- if (response == null) {
- // canceled successfuly
- LOG.warn("Got exception waiting resources for query master " + queryInProgress.getQueryId(), t);
- return null;
- }
- }
-
- if (response == null || response.getWorkerAllocatedResourceList().size() == 0) {
- return null;
- }
-
- WorkerAllocatedResource resource = response.getWorkerAllocatedResource(0);
- registerQueryMaster(queryInProgress.getQueryId(), resource.getContainerId());
- return resource;
- }
-
- private void registerQueryMaster(QueryId queryId, ContainerProtocol.TajoContainerIdProto containerId) {
- rmContext.getQueryMasterContainer().putIfAbsent(queryId, containerId);
- }
-
- @Override
- public void allocateWorkerResources(WorkerResourceAllocationRequest request,
- RpcCallback<WorkerResourceAllocationResponse> callBack) {
- try {
- //TODO checking queue size
- requestQueue.put(new WorkerResourceRequest(new QueryId(request.getQueryId()), false, request, callBack));
- } catch (InterruptedException e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- static class WorkerResourceRequest {
- boolean queryMasterRequest;
- QueryId queryId;
- WorkerResourceAllocationRequest request;
- RpcCallback<WorkerResourceAllocationResponse> callBack;
- WorkerResourceRequest(
- QueryId queryId,
- boolean queryMasterRequest, WorkerResourceAllocationRequest request,
- RpcCallback<WorkerResourceAllocationResponse> callBack) {
- this.queryId = queryId;
- this.queryMasterRequest = queryMasterRequest;
- this.request = request;
- this.callBack = callBack;
- }
- }
-
- static class AllocatedWorkerResource {
- Worker worker;
- int allocatedMemoryMB;
- float allocatedDiskSlots;
- }
-
- private static final long QUEUE_POLLING_TIME = 100;
-
- class WorkerResourceAllocationThread extends Thread {
- @Override
- public void run() {
- LOG.info("WorkerResourceAllocationThread start");
- while(!stopped.get()) {
- BasicFuture<ClusterResourceSummary> future = summaryRequest.expire();
- if (future != null) {
- future.done(makeClusterResourceSummary());
- }
- try {
- WorkerResourceRequest resourceRequest = requestQueue.poll(
- QUEUE_POLLING_TIME, TimeUnit.MILLISECONDS);
- if (resourceRequest == null) {
- continue;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("allocateWorkerResources:" +
- (new QueryId(resourceRequest.request.getQueryId())) +
- ", requiredMemory:" + resourceRequest.request.getMinMemoryMBPerContainer() +
- "~" + resourceRequest.request.getMaxMemoryMBPerContainer() +
- ", requiredContainers:" + resourceRequest.request.getNumContainers() +
- ", requiredDiskSlots:" + resourceRequest.request.getMinDiskSlotPerContainer() +
- "~" + resourceRequest.request.getMaxDiskSlotPerContainer() +
- ", queryMasterRequest=" + resourceRequest.queryMasterRequest +
- ", liveWorkers=" + rmContext.getWorkers().size());
- }
-
- // TajoWorkerResourceManager can't return allocated disk slots occasionally.
- // Because the rest resource request can remains after QueryMaster stops.
- // Thus we need to find whether QueryId stopped or not.
- if (!rmContext.getStoppedQueryIds().contains(resourceRequest.queryId)) {
- List<AllocatedWorkerResource> allocatedWorkerResources = chooseWorkers(resourceRequest);
-
- if(allocatedWorkerResources.size() > 0) {
- List<WorkerAllocatedResource> allocatedResources =
- new ArrayList<WorkerAllocatedResource>();
-
- for(AllocatedWorkerResource allocatedResource: allocatedWorkerResources) {
- NodeId nodeId = NodeId.newInstance(allocatedResource.worker.getConnectionInfo().getHost(),
- allocatedResource.worker.getConnectionInfo().getPeerRpcPort());
-
- TajoWorkerContainerId containerId = new TajoWorkerContainerId();
-
- containerId.setApplicationAttemptId(
- ApplicationIdUtils.createApplicationAttemptId(resourceRequest.queryId));
- containerId.setId(containerIdSeq.incrementAndGet());
-
- ContainerProtocol.TajoContainerIdProto containerIdProto = containerId.getProto();
- allocatedResources.add(WorkerAllocatedResource.newBuilder()
- .setContainerId(containerIdProto)
- .setConnectionInfo(allocatedResource.worker.getConnectionInfo().getProto())
- .setAllocatedMemoryMB(allocatedResource.allocatedMemoryMB)
- .setAllocatedDiskSlots(allocatedResource.allocatedDiskSlots)
- .build());
-
-
- allocatedResourceMap.putIfAbsent(containerIdProto, allocatedResource);
- }
-
- resourceRequest.callBack.run(WorkerResourceAllocationResponse.newBuilder()
- .setQueryId(resourceRequest.request.getQueryId())
- .addAllWorkerAllocatedResource(allocatedResources)
- .build()
- );
-
- } else {
- if(LOG.isDebugEnabled()) {
- LOG.debug("=========================================");
- LOG.debug("Available Workers");
- for(Worker worker: rmContext.getWorkers().values()) {
- LOG.debug(worker.toString());
- }
- LOG.debug("=========================================");
- }
- requestQueue.put(resourceRequest);
- Thread.sleep(QUEUE_POLLING_TIME);
- }
- }
- } catch(InterruptedException ie) {
- LOG.error(ie);
- } catch (Throwable t) {
- LOG.error(t, t);
- }
- }
- }
- }
-
- private static final long MAX_WAIT_TIME = 10000;
-
- public ClusterResourceSummary getClusterResourceSummary() {
- BasicFuture<ClusterResourceSummary> future =
- summaryRequest.check(new BasicFuture<ClusterResourceSummary>());
- try {
- return future.get(MAX_WAIT_TIME, TimeUnit.MILLISECONDS);
- } catch (Exception e) {
- LOG.warn("Failed to get cluster summary by exception", e);
- }
- return null;
- }
-
- private ClusterResourceSummary makeClusterResourceSummary() {
-
- int totalDiskSlots = 0;
- int totalCpuCoreSlots = 0;
- int totalMemoryMB = 0;
-
- int totalAvailableDiskSlots = 0;
- int totalAvailableCpuCoreSlots = 0;
- int totalAvailableMemoryMB = 0;
-
- for(Worker worker: rmContext.getWorkers().values()) {
-
- WorkerResource resource = worker.getResource();
-
- totalMemoryMB += resource.getMemoryMB();
- totalAvailableMemoryMB += resource.getAvailableMemoryMB();
-
- totalDiskSlots += resource.getDiskSlots();
- totalAvailableDiskSlots += resource.getAvailableDiskSlots();
-
- totalCpuCoreSlots += resource.getCpuCoreSlots();
- totalAvailableCpuCoreSlots += resource.getAvailableCpuCoreSlots();
- }
-
- return ClusterResourceSummary.newBuilder()
- .setNumWorkers(rmContext.getWorkers().size())
- .setTotalCpuCoreSlots(totalCpuCoreSlots)
- .setTotalDiskSlots(totalDiskSlots)
- .setTotalMemoryMB(totalMemoryMB)
- .setTotalAvailableCpuCoreSlots(totalAvailableCpuCoreSlots)
- .setTotalAvailableDiskSlots(totalAvailableDiskSlots)
- .setTotalAvailableMemoryMB(totalAvailableMemoryMB)
- .build();
- }
-
- private List<AllocatedWorkerResource> chooseWorkers(WorkerResourceRequest resourceRequest) {
- List<AllocatedWorkerResource> selectedWorkers = new ArrayList<AllocatedWorkerResource>();
-
- int allocatedResources = 0;
-
- ResourceRequestPriority resourceRequestPriority
- = resourceRequest.request.getResourceRequestPriority();
-
- List<Worker> randomWorkers = new ArrayList<Worker>(rmContext.getWorkers().values());
- Collections.shuffle(randomWorkers);
-
- if(resourceRequestPriority == ResourceRequestPriority.MEMORY) {
-
- int numContainers = resourceRequest.request.getNumContainers();
- int minMemoryMB = resourceRequest.request.getMinMemoryMBPerContainer();
- int maxMemoryMB = resourceRequest.request.getMaxMemoryMBPerContainer();
- float diskSlot = Math.max(resourceRequest.request.getMaxDiskSlotPerContainer(),
- resourceRequest.request.getMinDiskSlotPerContainer());
-
- int liveWorkerSize = randomWorkers.size();
- Set<Integer> insufficientWorkers = new HashSet<Integer>();
- boolean stop = false;
- boolean checkMax = true;
- while(!stop) {
- if(allocatedResources >= numContainers) {
- break;
- }
-
- if(insufficientWorkers.size() >= liveWorkerSize) {
- if(!checkMax) {
- break;
- }
- insufficientWorkers.clear();
- checkMax = false;
- }
- int compareAvailableMemory = checkMax ? maxMemoryMB : minMemoryMB;
-
- for(Worker worker: randomWorkers) {
- if(allocatedResources >= numContainers) {
- stop = true;
- break;
- }
-
- if(insufficientWorkers.size() >= liveWorkerSize) {
- break;
- }
-
- WorkerResource workerResource = worker.getResource();
- if(workerResource.getAvailableMemoryMB() >= compareAvailableMemory) {
- int workerMemory;
- if(workerResource.getAvailableMemoryMB() >= maxMemoryMB) {
- workerMemory = maxMemoryMB;
- } else {
- workerMemory = workerResource.getAvailableMemoryMB();
- }
- AllocatedWorkerResource allocatedWorkerResource = new AllocatedWorkerResource();
- allocatedWorkerResource.worker = worker;
- allocatedWorkerResource.allocatedMemoryMB = workerMemory;
- if(workerResource.getAvailableDiskSlots() >= diskSlot) {
- allocatedWorkerResource.allocatedDiskSlots = diskSlot;
- } else {
- allocatedWorkerResource.allocatedDiskSlots = workerResource.getAvailableDiskSlots();
- }
-
- workerResource.allocateResource(allocatedWorkerResource.allocatedDiskSlots,
- allocatedWorkerResource.allocatedMemoryMB);
-
- selectedWorkers.add(allocatedWorkerResource);
-
- allocatedResources++;
- } else {
- insufficientWorkers.add(worker.getWorkerId());
- }
- }
- }
- } else {
- int numContainers = resourceRequest.request.getNumContainers();
- float minDiskSlots = resourceRequest.request.getMinDiskSlotPerContainer();
- float maxDiskSlots = resourceRequest.request.getMaxDiskSlotPerContainer();
- int memoryMB = Math.max(resourceRequest.request.getMaxMemoryMBPerContainer(),
- resourceRequest.request.getMinMemoryMBPerContainer());
-
- int liveWorkerSize = randomWorkers.size();
- Set<Integer> insufficientWorkers = new HashSet<Integer>();
- boolean stop = false;
- boolean checkMax = true;
- while(!stop) {
- if(allocatedResources >= numContainers) {
- break;
- }
-
- if(insufficientWorkers.size() >= liveWorkerSize) {
- if(!checkMax) {
- break;
- }
- insufficientWorkers.clear();
- checkMax = false;
- }
- float compareAvailableDisk = checkMax ? maxDiskSlots : minDiskSlots;
-
- for(Worker worker: randomWorkers) {
- if(allocatedResources >= numContainers) {
- stop = true;
- break;
- }
-
- if(insufficientWorkers.size() >= liveWorkerSize) {
- break;
- }
-
- WorkerResource workerResource = worker.getResource();
- if(workerResource.getAvailableDiskSlots() >= compareAvailableDisk) {
- float workerDiskSlots;
- if(workerResource.getAvailableDiskSlots() >= maxDiskSlots) {
- workerDiskSlots = maxDiskSlots;
- } else {
- workerDiskSlots = workerResource.getAvailableDiskSlots();
- }
- AllocatedWorkerResource allocatedWorkerResource = new AllocatedWorkerResource();
- allocatedWorkerResource.worker = worker;
- allocatedWorkerResource.allocatedDiskSlots = workerDiskSlots;
-
- if(workerResource.getAvailableMemoryMB() >= memoryMB) {
- allocatedWorkerResource.allocatedMemoryMB = memoryMB;
- } else {
- allocatedWorkerResource.allocatedMemoryMB = workerResource.getAvailableMemoryMB();
- }
- workerResource.allocateResource(allocatedWorkerResource.allocatedDiskSlots,
- allocatedWorkerResource.allocatedMemoryMB);
-
- selectedWorkers.add(allocatedWorkerResource);
-
- allocatedResources++;
- } else {
- insufficientWorkers.add(worker.getWorkerId());
- }
- }
- }
- }
- return selectedWorkers;
- }
-
- /**
- * Release allocated resource.
- *
- * @param containerId ContainerIdProto to be released
- */
- @Override
- public void releaseWorkerResource(ContainerProtocol.TajoContainerIdProto containerId) {
- AllocatedWorkerResource allocated = allocatedResourceMap.remove(containerId);
- if(allocated != null) {
- LOG.info("Release Resource: " + allocated.allocatedDiskSlots + "," + allocated.allocatedMemoryMB);
- allocated.worker.getResource().releaseResource( allocated.allocatedDiskSlots, allocated.allocatedMemoryMB);
- } else {
- LOG.warn("No AllocatedWorkerResource data for [" + containerId + "]");
- return;
- }
- }
-
- @Override
- public boolean isQueryMasterStopped(QueryId queryId) {
- return !rmContext.getQueryMasterContainer().containsKey(queryId);
- }
-
- @Override
- public void releaseQueryMaster(QueryId queryId) {
- if(!rmContext.getQueryMasterContainer().containsKey(queryId)) {
- LOG.warn("No QueryMaster resource info for " + queryId);
- return;
- } else {
- ContainerProtocol.TajoContainerIdProto containerId = rmContext.getQueryMasterContainer().remove(queryId);
- releaseWorkerResource(containerId);
- rmContext.getStoppedQueryIds().add(queryId);
- LOG.info(String.format("Released QueryMaster (%s) resource." , queryId.toString()));
- }
- }
-
- public TajoRMContext getRMContext() {
- return rmContext;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/Worker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/Worker.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/Worker.java
deleted file mode 100644
index 6535688..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/Worker.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.state.*;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-
-import java.util.EnumSet;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * It contains resource and various information for a worker.
- */
-public class Worker implements EventHandler<WorkerEvent>, Comparable<Worker> {
- /** class logger */
- private static final Log LOG = LogFactory.getLog(Worker.class);
-
- private final ReentrantReadWriteLock.ReadLock readLock;
- private final ReentrantReadWriteLock.WriteLock writeLock;
-
- /** context of {@link org.apache.tajo.master.rm.TajoWorkerResourceManager} */
- private final TajoRMContext rmContext;
-
- /** last heartbeat time */
- private long lastHeartbeatTime;
-
- /** Resource capability */
- private final WorkerResource resource;
-
- /** Worker connection information */
- private WorkerConnectionInfo connectionInfo;
-
- private static final ReconnectNodeTransition RECONNECT_NODE_TRANSITION = new ReconnectNodeTransition();
- private static final StatusUpdateTransition STATUS_UPDATE_TRANSITION = new StatusUpdateTransition();
-
- private static final StateMachineFactory<Worker,
- WorkerState,
- WorkerEventType,
- WorkerEvent> stateMachineFactory
- = new StateMachineFactory<Worker,
- WorkerState,
- WorkerEventType,
- WorkerEvent>(WorkerState.NEW)
-
- // Transition from NEW
- .addTransition(WorkerState.NEW, WorkerState.RUNNING,
- WorkerEventType.STARTED,
- new AddNodeTransition())
-
- // Transition from RUNNING
- .addTransition(WorkerState.RUNNING, EnumSet.of(WorkerState.RUNNING, WorkerState.UNHEALTHY),
- WorkerEventType.STATE_UPDATE,
- STATUS_UPDATE_TRANSITION)
- .addTransition(WorkerState.RUNNING, WorkerState.LOST,
- WorkerEventType.EXPIRE,
- new DeactivateNodeTransition(WorkerState.LOST))
- .addTransition(WorkerState.RUNNING, WorkerState.RUNNING,
- WorkerEventType.RECONNECTED,
- RECONNECT_NODE_TRANSITION)
-
- // Transitions from UNHEALTHY state
- .addTransition(WorkerState.UNHEALTHY, EnumSet.of(WorkerState.RUNNING, WorkerState.UNHEALTHY),
- WorkerEventType.STATE_UPDATE,
- STATUS_UPDATE_TRANSITION)
- .addTransition(WorkerState.UNHEALTHY, WorkerState.LOST,
- WorkerEventType.EXPIRE,
- new DeactivateNodeTransition(WorkerState.LOST))
- .addTransition(WorkerState.UNHEALTHY, WorkerState.UNHEALTHY,
- WorkerEventType.RECONNECTED,
- RECONNECT_NODE_TRANSITION);
-
- private final StateMachine<WorkerState, WorkerEventType, WorkerEvent> stateMachine =
- stateMachineFactory.make(this, WorkerState.NEW);
-
- public Worker(TajoRMContext rmContext, WorkerResource resource, WorkerConnectionInfo connectionInfo) {
- this.rmContext = rmContext;
-
- this.connectionInfo = connectionInfo;
- this.lastHeartbeatTime = System.currentTimeMillis();
- this.resource = resource;
-
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- this.readLock = lock.readLock();
- this.writeLock = lock.writeLock();
- }
-
- public int getWorkerId() {
- return connectionInfo.getId();
- }
-
- public WorkerConnectionInfo getConnectionInfo() {
- return connectionInfo;
- }
-
- public void setLastHeartbeatTime(long lastheartbeatReportTime) {
- this.writeLock.lock();
-
- try {
- this.lastHeartbeatTime = lastheartbeatReportTime;
- } finally {
- this.writeLock.unlock();
- }
- }
-
- public long getLastHeartbeatTime() {
- this.readLock.lock();
-
- try {
- return this.lastHeartbeatTime;
- } finally {
- this.readLock.unlock();
- }
- }
-
- /**
- *
- * @return the current state of worker
- */
- public WorkerState getState() {
- this.readLock.lock();
-
- try {
- return this.stateMachine.getCurrentState();
- } finally {
- this.readLock.unlock();
- }
- }
-
- /**
- *
- * @return the current resource capability of worker
- */
- public WorkerResource getResource() {
- return this.resource;
- }
-
- @Override
- public int compareTo(Worker o) {
- if(o == null) {
- return 1;
- }
- return connectionInfo.compareTo(o.connectionInfo);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- Worker worker = (Worker) o;
-
- if (lastHeartbeatTime != worker.lastHeartbeatTime) return false;
- if (connectionInfo != null ? !connectionInfo.equals(worker.connectionInfo) : worker.connectionInfo != null)
- return false;
- if (readLock != null ? !readLock.equals(worker.readLock) : worker.readLock != null) return false;
- if (resource != null ? !resource.equals(worker.resource) : worker.resource != null) return false;
- if (rmContext != null ? !rmContext.equals(worker.rmContext) : worker.rmContext != null) return false;
- if (stateMachine != null ? !stateMachine.equals(worker.stateMachine) : worker.stateMachine != null) return false;
- if (writeLock != null ? !writeLock.equals(worker.writeLock) : worker.writeLock != null) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = readLock != null ? readLock.hashCode() : 0;
- result = 31 * result + (writeLock != null ? writeLock.hashCode() : 0);
- result = 31 * result + (rmContext != null ? rmContext.hashCode() : 0);
- result = 31 * result + (int) (lastHeartbeatTime ^ (lastHeartbeatTime >>> 32));
- result = 31 * result + (resource != null ? resource.hashCode() : 0);
- result = 31 * result + (connectionInfo != null ? connectionInfo.hashCode() : 0);
- result = 31 * result + (stateMachine != null ? stateMachine.hashCode() : 0);
- return result;
- }
-
- public static class AddNodeTransition implements SingleArcTransition<Worker, WorkerEvent> {
- @Override
- public void transition(Worker worker, WorkerEvent workerEvent) {
-
- worker.rmContext.getQueryMasterWorker().add(worker.getWorkerId());
- LOG.info("Worker with " + worker.getResource() + " is joined to Tajo cluster");
- }
- }
-
- public static class StatusUpdateTransition implements
- MultipleArcTransition<Worker, WorkerEvent, WorkerState> {
-
- @Override
- public WorkerState transition(Worker worker, WorkerEvent event) {
- if (!(event instanceof WorkerStatusEvent)) {
- throw new IllegalArgumentException("event should be a WorkerStatusEvent type.");
- }
- WorkerStatusEvent statusEvent = (WorkerStatusEvent) event;
- worker.updateStatus(statusEvent);
-
- return WorkerState.RUNNING;
- }
- }
-
- private void updateStatus(WorkerStatusEvent statusEvent) {
- this.writeLock.lock();
-
- try {
- lastHeartbeatTime = System.currentTimeMillis();
- resource.setNumRunningTasks(statusEvent.getRunningTaskNum());
- resource.setMaxHeap(statusEvent.maxHeap());
- resource.setFreeHeap(statusEvent.getFreeHeap());
- resource.setTotalHeap(statusEvent.getTotalHeap());
- } finally {
- this.writeLock.unlock();
- }
- }
-
- public static class DeactivateNodeTransition implements SingleArcTransition<Worker, WorkerEvent> {
- private final WorkerState finalState;
-
- public DeactivateNodeTransition(WorkerState finalState) {
- this.finalState = finalState;
- }
-
- @Override
- public void transition(Worker worker, WorkerEvent workerEvent) {
-
- worker.rmContext.getWorkers().remove(worker.getWorkerId());
- LOG.info("Deactivating Node " + worker.getWorkerId() + " as it is now " + finalState);
- worker.rmContext.getInactiveWorkers().putIfAbsent(worker.getWorkerId(), worker);
- }
- }
-
- public static class ReconnectNodeTransition implements SingleArcTransition<Worker, WorkerEvent> {
-
- @Override
- public void transition(Worker worker, WorkerEvent workerEvent) {
- if (!(workerEvent instanceof WorkerReconnectEvent)) {
- throw new IllegalArgumentException("workerEvent should be a WorkerReconnectEvent type.");
- }
- WorkerReconnectEvent castedEvent = (WorkerReconnectEvent) workerEvent;
-
- Worker newWorker = castedEvent.getWorker();
- worker.rmContext.getWorkers().put(castedEvent.getWorkerId(), newWorker);
- worker.rmContext.getDispatcher().getEventHandler().handle(
- new WorkerEvent(worker.getWorkerId(), WorkerEventType.STARTED));
- }
- }
-
- @Override
- public void handle(WorkerEvent event) {
- LOG.debug("Processing " + event.getWorkerId() + " of type " + event.getType());
- try {
- writeLock.lock();
- WorkerState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitonException e) {
- LOG.error("Can't handle this event at current state"
- + ", eventType:" + event.getType().name()
- + ", oldState:" + oldState.name()
- + ", nextState:" + getState().name()
- , e);
- LOG.error("Invalid event " + event.getType() + " on Worker " + getWorkerId());
- }
- if (oldState != getState()) {
- LOG.info(getWorkerId() + " Node Transitioned from " + oldState + " to " + getState());
- }
- }
-
- finally {
- writeLock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEvent.java
deleted file mode 100644
index c208990..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-
-/**
- * WorkerEvent describes all kinds of events which sent to {@link Worker}.
- */
-public class WorkerEvent extends AbstractEvent<WorkerEventType> {
- private final int workerId;
-
- public WorkerEvent(int workerId, WorkerEventType workerEventType) {
- super(workerEventType);
- this.workerId = workerId;
- }
-
- public int getWorkerId() {
- return workerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEventType.java
deleted file mode 100644
index 0c97654..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerEventType.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-public enum WorkerEventType {
-
- /** Source : {@link TajoResourceTracker}, Destination: {@link Worker} */
- STARTED,
- STATE_UPDATE,
- RECONNECTED,
-
- /** Source : {@link WorkerLivelinessMonitor}, Destination: {@link Worker} */
- EXPIRE
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerLivelinessMonitor.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerLivelinessMonitor.java
deleted file mode 100644
index 2751886..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerLivelinessMonitor.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.util.AbstractLivelinessMonitor;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tajo.conf.TajoConf;
-
-/**
- * It periodically checks the latest heartbeat time of {@link Worker}.
- * If the latest heartbeat time is expired, it produces EXPIRE event to a corresponding {@link Worker}.
- */
-public class WorkerLivelinessMonitor extends AbstractLivelinessMonitor<Integer> {
-
- private EventHandler dispatcher;
-
- public WorkerLivelinessMonitor(Dispatcher d) {
- super(WorkerLivelinessMonitor.class.getSimpleName(), new SystemClock());
- this.dispatcher = d.getEventHandler();
- }
-
- public void serviceInit(Configuration conf) throws Exception {
- Preconditions.checkArgument(conf instanceof TajoConf);
- TajoConf systemConf = (TajoConf) conf;
- // milliseconds
- int expireIntvl = systemConf.getIntVar(TajoConf.ConfVars.RESOURCE_TRACKER_HEARTBEAT_TIMEOUT);
- setExpireInterval(expireIntvl);
- setMonitorInterval(expireIntvl/3);
- super.serviceInit(conf);
- }
-
- @Override
- protected void expire(Integer id) {
- dispatcher.handle(new WorkerEvent(id, WorkerEventType.EXPIRE));
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerReconnectEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerReconnectEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerReconnectEvent.java
deleted file mode 100644
index 3828b6a..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerReconnectEvent.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-/**
- * {@link TajoResourceTracker} produces this event, and it's destination is {@link Worker}.
- * This event occurs only when an inactive worker sends a ping again.
- */
-public class WorkerReconnectEvent extends WorkerEvent {
- private final Worker worker;
- public WorkerReconnectEvent(int workerId, Worker worker) {
- super(workerId, WorkerEventType.RECONNECTED);
- this.worker = worker;
- }
-
- public Worker getWorker() {
- return worker;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
deleted file mode 100644
index 3d5e062..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import com.google.protobuf.RpcCallback;
-import org.apache.hadoop.service.Service;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.ClusterResourceSummary;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.WorkerResourceAllocationRequest;
-import org.apache.tajo.master.QueryInProgress;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-/**
- * An interface of WorkerResourceManager which allows TajoMaster to request allocation for containers
- * and release the allocated containers.
- */
-public interface WorkerResourceManager extends Service {
-
- /**
- * Request a resource container for a QueryMaster.
- *
- * @param queryInProgress QueryInProgress
- * @return A allocated container resource
- */
- @Deprecated
- public QueryCoordinatorProtocol.WorkerAllocatedResource allocateQueryMaster(QueryInProgress queryInProgress);
-
- /**
- * Request one or more resource containers. You can set the number of containers and resource capabilities, such as
- * memory, CPU cores, and disk slots. This is an asynchronous call. You should use a callback to get allocated
- * resource containers. Each container is identified {@link org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto}.
- *
- * @param request Request description
- * @param rpcCallBack Callback function
- */
- public void allocateWorkerResources(WorkerResourceAllocationRequest request,
- RpcCallback<QueryCoordinatorProtocol.WorkerResourceAllocationResponse> rpcCallBack);
-
- /**
- * Release a container
- *
- * @param containerId ContainerIdProto to be released
- */
- public void releaseWorkerResource(ContainerProtocol.TajoContainerIdProto containerId);
-
- public String getSeedQueryId() throws IOException;
-
- /**
- * Check if a query master is stopped.
- *
- * @param queryId QueryId to be checked
- * @return True if QueryMaster is stopped
- */
- public boolean isQueryMasterStopped(QueryId queryId);
-
- /**
- * Stop a query master
- *
- * @param queryId QueryId to be stopped
- */
- public void releaseQueryMaster(QueryId queryId);
-
- /**
- *
- * @return a Map instance containing active workers
- */
- public Map<Integer, Worker> getWorkers();
-
- /**
- *
- * @return a Map instance containing inactive workers
- */
- public Map<Integer, Worker> getInactiveWorkers();
-
- public void stop();
-
- /**
- *
- * @return The overall summary of cluster resources
- */
- public ClusterResourceSummary getClusterResourceSummary();
-
- /**
- *
- * @return WorkerIds on which QueryMasters are running
- */
- public Collection<Integer> getQueryMasters();
-
- /**
- *
- * @return RMContext
- */
- public TajoRMContext getRMContext();
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerState.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerState.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerState.java
deleted file mode 100644
index a941008..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerState.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-/**
- * It presents the states of {@link Worker}.
- */
-public enum WorkerState {
- /** New worker */
- NEW,
-
- /** Running worker */
- RUNNING,
-
- /** Worker is unhealthy */
- UNHEALTHY,
-
- /** worker is out of service */
- DECOMMISSIONED,
-
- /** worker has not sent a heartbeat for some configured time threshold */
- LOST;
-
- @SuppressWarnings("unused")
- public boolean isUnusable() {
- return (this == UNHEALTHY || this == DECOMMISSIONED || this == LOST);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerStatusEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerStatusEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerStatusEvent.java
deleted file mode 100644
index f1ab401..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerStatusEvent.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-/**
- * {@link TajoResourceTracker} produces this event, and its destination is
- * {@link org.apache.tajo.master.rm.Worker.StatusUpdateTransition} of {@link Worker}.
- */
-public class WorkerStatusEvent extends WorkerEvent {
- private final int runningTaskNum;
- private final long maxHeap;
- private final long freeHeap;
- private final long totalHeap;
-
- public WorkerStatusEvent(int workerId, int runningTaskNum, long maxHeap, long freeHeap, long totalHeap) {
- super(workerId, WorkerEventType.STATE_UPDATE);
- this.runningTaskNum = runningTaskNum;
- this.maxHeap = maxHeap;
- this.freeHeap = freeHeap;
- this.totalHeap = totalHeap;
- }
-
- public int getRunningTaskNum() {
- return runningTaskNum;
- }
-
- public long maxHeap() {
- return maxHeap;
- }
-
- public long getFreeHeap() {
- return freeHeap;
- }
-
- public long getTotalHeap() {
- return totalHeap;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/AbstractQueryScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/AbstractQueryScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/AbstractQueryScheduler.java
new file mode 100644
index 0000000..1380417
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/AbstractQueryScheduler.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.resource.ResourceCalculator;
+
+/**
+ * please refer to {@TajoResourceScheduler} for detailed information.
+ */
+public abstract class AbstractQueryScheduler extends AbstractService implements TajoResourceScheduler {
+
+ protected final NodeResource clusterResource;
+ protected final NodeResource minResource;
+ protected final NodeResource maxResource;
+ protected final NodeResource qmMinResource;
+
+ public AbstractQueryScheduler(String name) {
+ super(name);
+ this.minResource = NodeResources.createResource(0);
+ this.qmMinResource = NodeResources.createResource(0);
+ this.maxResource = NodeResources.createResource(0);
+ this.clusterResource = NodeResources.createResource(0);
+ }
+
+ @Override
+ public NodeResource getClusterResource() {
+ return clusterResource;
+ }
+
+ @Override
+ public NodeResource getMinimumResourceCapability() {
+ return minResource;
+ }
+
+ @Override
+ public NodeResource getMaximumResourceCapability() {
+ return maxResource;
+ }
+
+ @Override
+ public NodeResource getQMMinimumResourceCapability() {
+ return qmMinResource;
+ }
+
+ public abstract int getRunningQuery();
+
+ public abstract ResourceCalculator getResourceCalculator();
+
+ public abstract void submitQuery(QuerySchedulingInfo schedulingInfo);
+
+ public abstract void stopQuery(QueryId queryId);
+
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QuerySchedulingInfo.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QuerySchedulingInfo.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QuerySchedulingInfo.java
index 3dd3389..a1fe743 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QuerySchedulingInfo.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QuerySchedulingInfo.java
@@ -20,13 +20,27 @@ package org.apache.tajo.master.scheduler;
import com.google.common.base.Objects;
import org.apache.tajo.QueryId;
+import org.apache.tajo.util.NumberUtil;
-public class QuerySchedulingInfo {
+/**
+ * A QuerySchedulingInfo represents an scheduling information.
+ * It provides a common interface for queue and priority
+ */
+
+public class QuerySchedulingInfo implements Comparable<QuerySchedulingInfo> {
+ /** Name of queue */
+ private String queue;
+ /** Query owner */
+ private String user;
private QueryId queryId;
- private Integer priority;
- private Long startTime;
+ /** Query priority for queries in same queue */
+ private int priority;
+ /** Start time for query in same queue */
+ private long startTime;
- public QuerySchedulingInfo(QueryId queryId, Integer priority, Long startTime) {
+ public QuerySchedulingInfo(String queue, String user, QueryId queryId, int priority, long startTime) {
+ this.queue = queue;
+ this.user = user;
this.queryId = queryId;
this.priority = priority;
this.startTime = startTime;
@@ -36,11 +50,15 @@ public class QuerySchedulingInfo {
return queryId;
}
- public Integer getPriority() {
+ public String getUser() {
+ return user;
+ }
+
+ public int getPriority() {
return priority;
}
- public Long getStartTime() {
+ public long getStartTime() {
return startTime;
}
@@ -48,8 +66,36 @@ public class QuerySchedulingInfo {
return queryId.getId();
}
+ public String getQueue() {
+ return queue;
+ }
+
+
+ @Override
+ public int compareTo(QuerySchedulingInfo o) {
+ int ret = NumberUtil.compare(priority, o.priority);
+ if(ret == 0) {
+ ret = NumberUtil.compare(startTime, o.startTime);
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ QuerySchedulingInfo other = (QuerySchedulingInfo) obj;
+ if (!this.getQueryId().equals(other.getQueryId()))
+ return false;
+ return true;
+ }
+
@Override
public int hashCode() {
- return Objects.hashCode(startTime, getName(), priority);
+ return Objects.hashCode(queryId, queue, user, priority, startTime);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueInfo.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueInfo.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueInfo.java
new file mode 100644
index 0000000..acf793c
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueInfo.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import java.lang.String;import java.util.List;
+
+/**
+ * QueueInfo is a report of the runtime information of the queue.
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>Queue name.</li>
+ * <li>Capacity of the queue.</li>
+ * <li>Maximum capacity of the queue.</li>
+ * <li>Current capacity of the queue.</li>
+ * <li>Child queues.</li>
+ * <li>Running applications.</li>
+ * <li>{@link QueueState} of the queue.</li>
+ * </ul>
+ *
+ */
+
+public abstract class QueueInfo {
+ /**
+ * Get the <em>name</em> of the queue.
+ * @return <em>name</em> of the queue
+ */
+ public abstract String getQueueName();
+
+ public abstract void setQueueName(String queueName);
+
+ /**
+ * Get the <em>configured capacity</em> of the queue.
+ * @return <em>configured capacity</em> of the queue
+ */
+ public abstract float getCapacity();
+
+ public abstract void setCapacity(float capacity);
+
+ /**
+ * Get the <em>maximum capacity</em> of the queue.
+ * @return <em>maximum capacity</em> of the queue
+ */
+
+ public abstract float getMaximumCapacity();
+
+ public abstract void setMaximumCapacity(float maximumCapacity);
+
+ /**
+ * Get the <em>maximum query capacity</em> of the queue.
+ * @return <em>maximum query capacity</em> of the queue
+ */
+
+ public abstract float getMaximumQueryCapacity();
+
+ public abstract void setMaximumQueryCapacity(float maximumQueryCapacity);
+
+ /**
+ * Get the <em>current capacity</em> of the queue.
+ * @return <em>current capacity</em> of the queue
+ */
+
+ public abstract float getCurrentCapacity();
+
+ public abstract void setCurrentCapacity(float currentCapacity);
+
+ /**
+ * Get the <em>child queues</em> of the queue.
+ * @return <em>child queues</em> of the queue
+ */
+
+ public abstract List<QueueInfo> getChildQueues();
+
+ public abstract void setChildQueues(List<QueueInfo> childQueues);
+
+
+ /**
+ * Get the <code>QueueState</code> of the queue.
+ * @return <code>QueueState</code> of the queue
+ */
+ public abstract QueueState getQueueState();
+
+ public abstract void setQueueState(QueueState queueState);
+
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueState.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueState.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueState.java
new file mode 100644
index 0000000..d7acbfe
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/QueueState.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+
+/**
+ * State of a Queue.
+ * <p>
+ * A queue is in one of:
+ * <ul>
+ * <li>{@link #RUNNING} - normal state.</li>
+ * <li>{@link #STOPPED} - not accepting new application submissions.</li>
+ * </ul>
+ *
+ * @see QueueInfo
+ * @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)
+ */
+@Public
+@Stable
+public enum QueueState {
+ /**
+ * Stopped - Not accepting submissions of new applications.
+ */
+ STOPPED,
+
+ /**
+ * Running - normal operation.
+ */
+ RUNNING
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SchedulingAlgorithms.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SchedulingAlgorithms.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SchedulingAlgorithms.java
index 7fd07b5..efe4561 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SchedulingAlgorithms.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SchedulingAlgorithms.java
@@ -18,6 +18,8 @@
package org.apache.tajo.master.scheduler;
+import org.apache.tajo.util.NumberUtil;
+
import java.util.Comparator;
/**
@@ -32,9 +34,9 @@ public class SchedulingAlgorithms {
public static class FifoComparator implements Comparator<QuerySchedulingInfo> {
@Override
public int compare(QuerySchedulingInfo q1, QuerySchedulingInfo q2) {
- int res = q1.getPriority().compareTo(q2.getPriority());
+ int res = NumberUtil.compare(q1.getPriority(), q2.getPriority());
if (res == 0) {
- res = (int) Math.signum(q1.getStartTime() - q2.getStartTime());
+ res = NumberUtil.compare(q1.getStartTime(), q2.getStartTime());
}
if (res == 0) {
// In the rare case where jobs were submitted at the exact same time,
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleFifoScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleFifoScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleFifoScheduler.java
deleted file mode 100644
index 6cb98eb..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleFifoScheduler.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.scheduler;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.master.QueryInProgress;
-import org.apache.tajo.master.QueryManager;
-
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class SimpleFifoScheduler implements Scheduler {
- private static final Log LOG = LogFactory.getLog(SimpleFifoScheduler.class.getName());
- private LinkedList<QuerySchedulingInfo> pool = new LinkedList<QuerySchedulingInfo>();
- private final Thread queryProcessor;
- private AtomicBoolean stopped = new AtomicBoolean();
- private QueryManager manager;
- private Comparator<QuerySchedulingInfo> COMPARATOR = new SchedulingAlgorithms.FifoComparator();
-
- public SimpleFifoScheduler(QueryManager manager) {
- this.manager = manager;
- this.queryProcessor = new Thread(new QueryProcessor());
- this.queryProcessor.setName("Query Processor");
- }
-
- @Override
- public Mode getMode() {
- return Mode.FIFO;
- }
-
- @Override
- public String getName() {
- return manager.getName();
- }
-
- @Override
- public boolean addQuery(QueryInProgress queryInProgress) {
- int qSize = pool.size();
- if (qSize != 0 && qSize % 100 == 0) {
- LOG.info("Size of Fifo queue is " + qSize);
- }
-
- QuerySchedulingInfo querySchedulingInfo = new QuerySchedulingInfo(queryInProgress.getQueryId(), 1,
- queryInProgress.getQueryInfo().getStartTime());
- boolean result = pool.add(querySchedulingInfo);
- if (getRunningQueries().size() == 0) wakeupProcessor();
- return result;
- }
-
- @Override
- public boolean removeQuery(QueryId queryId) {
- return pool.remove(getQueryByQueryId(queryId));
- }
-
- public QuerySchedulingInfo getQueryByQueryId(QueryId queryId) {
- for (QuerySchedulingInfo querySchedulingInfo : pool) {
- if (querySchedulingInfo.getQueryId().equals(queryId)) {
- return querySchedulingInfo;
- }
- }
- return null;
- }
-
- @Override
- public List<QueryInProgress> getRunningQueries() {
- return new ArrayList<QueryInProgress>(manager.getRunningQueries());
- }
-
- public void start() {
- queryProcessor.start();
- }
-
- public void stop() {
- if (stopped.getAndSet(true)) {
- return;
- }
- pool.clear();
- synchronized (queryProcessor) {
- queryProcessor.interrupt();
- }
- }
-
- private QuerySchedulingInfo pollScheduledQuery() {
- if (pool.size() > 1) {
- Collections.sort(pool, COMPARATOR);
- }
- return pool.poll();
- }
-
- private void wakeupProcessor() {
- synchronized (queryProcessor) {
- queryProcessor.notifyAll();
- }
- }
-
- private final class QueryProcessor implements Runnable {
- @Override
- public void run() {
-
- QuerySchedulingInfo query;
-
- while (!stopped.get() && !Thread.currentThread().isInterrupted()) {
- query = null;
- if (getRunningQueries().size() == 0) {
- query = pollScheduledQuery();
- }
-
- if (query != null) {
- try {
- manager.startQueryJob(query.getQueryId());
- } catch (Throwable t) {
- LOG.fatal("Exception during query startup:", t);
- manager.stopQuery(query.getQueryId());
- }
- }
-
- synchronized (queryProcessor) {
- try {
- queryProcessor.wait(500);
- } catch (InterruptedException e) {
- if (stopped.get()) {
- break;
- }
- LOG.warn("Exception during shutdown: ", e);
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java
new file mode 100644
index 0000000..e41ac95
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.master.QueryInfo;
+import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.rm.TajoRMContext;
+import org.apache.tajo.master.rm.NodeStatus;
+import org.apache.tajo.master.scheduler.event.ResourceReserveSchedulerEvent;
+import org.apache.tajo.master.scheduler.event.SchedulerEvent;
+import org.apache.tajo.resource.DefaultResourceCalculator;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.resource.ResourceCalculator;
+import org.apache.tajo.util.TUtil;
+
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
+
+import static org.apache.tajo.ResourceProtos.*;
+
+/**
+ * SimpleScheduler can execute query and stages simultaneously.
+ * Each query and the stage competes to get the resource
+ */
+public class SimpleScheduler extends AbstractQueryScheduler {
+
+ private static final Log LOG = LogFactory.getLog(SimpleScheduler.class);
+ private static final float MAXIMUM_RUNNING_QM_RATE = 0.5f;
+ private static final Comparator<QuerySchedulingInfo> COMPARATOR = new SchedulingAlgorithms.FifoComparator();
+
+ private volatile boolean isStopped = false;
+ private final TajoMaster.MasterContext masterContext;
+
+ private final TajoRMContext rmContext;
+ private final BlockingQueue<QuerySchedulingInfo> queryQueue;
+ private final Map<QueryId, QuerySchedulingInfo> pendingQueryMap = Maps.newHashMap();
+
+ private final Map<QueryId, Integer> assignedQueryMasterMap = Maps.newHashMap();
+ private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
+
+ private final Thread queryProcessor;
+ private TajoConf tajoConf;
+
+ @VisibleForTesting
+ public SimpleScheduler(TajoMaster.MasterContext context, TajoRMContext rmContext) {
+ super(SimpleScheduler.class.getName());
+ this.masterContext = context;
+ this.rmContext = rmContext;
+ //Copy default array capacity from PriorityBlockingQueue.
+ this.queryQueue = new PriorityBlockingQueue<QuerySchedulingInfo>(11, COMPARATOR);
+ this.queryProcessor = new Thread(new QueryProcessor());
+ }
+
+ public SimpleScheduler(TajoMaster.MasterContext context) {
+ this(context, context.getResourceManager().getRMContext());
+ }
+
+ private void initScheduler(TajoConf conf) {
+ this.minResource.setMemory(conf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY)).setVirtualCores(1);
+ this.qmMinResource.setMemory(conf.getIntVar(TajoConf.ConfVars.QUERYMASTER_MINIMUM_MEMORY)).setVirtualCores(1);
+ updateResource();
+ this.queryProcessor.setName("Query Processor");
+ }
+
+ private void updateResource() {
+ NodeResource resource = NodeResources.createResource(0);
+ NodeResource totalResource = NodeResources.createResource(0);
+ for (NodeStatus nodeStatus : getRMContext().getNodes().values()) {
+ NodeResources.addTo(resource, nodeStatus.getAvailableResource());
+ NodeResources.addTo(totalResource, nodeStatus.getTotalResourceCapability());
+
+ }
+
+ NodeResources.update(maxResource, totalResource);
+ NodeResources.update(clusterResource, resource);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Cluster Resource. available : " + getClusterResource()
+ + " maximum: " + getMaximumResourceCapability());
+ }
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+ initScheduler(tajoConf);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ this.queryProcessor.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ this.isStopped = true;
+ super.serviceStop();
+ }
+
+ @Override
+ public int getRunningQuery() {
+ return assignedQueryMasterMap.size();
+ }
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return resourceCalculator;
+ }
+
+ private NodeResourceRequest createQMResourceRequest(QueryInfo queryInfo) {
+ NodeResource qmResource = getQMMinimumResourceCapability();
+
+ int containers = 1;
+ Set<Integer> assignedQMNodes = Sets.newHashSet(assignedQueryMasterMap.values());
+ List<Integer> idleNode = Lists.newArrayList();
+
+ for (NodeStatus nodeStatus : getRMContext().getNodes().values()) {
+
+ //find idle node for QM
+ if (!assignedQMNodes.contains(nodeStatus.getWorkerId())) {
+ idleNode.add(nodeStatus.getWorkerId());
+ }
+
+ if (idleNode.size() > containers * 3) break;
+ }
+
+ NodeResourceRequest.Builder builder = NodeResourceRequest.newBuilder();
+
+ builder.setQueryId(queryInfo.getQueryId().getProto())
+ .setCapacity(qmResource.getProto())
+ .setType(ResourceType.QUERYMASTER)
+ .setPriority(1)
+ .setNumContainers(containers)
+ .setRunningTasks(1)
+ .addAllCandidateNodes(idleNode)
+ .setUserId(queryInfo.getQueryContext().getUser());
+ //TODO .setQueue(queryInfo.getQueue());
+ return builder.build();
+ }
+
+
+ @Override
+ public int getNumClusterNodes() {
+ return rmContext.getNodes().size();
+ }
+
+ @Override
+ public List<AllocationResourceProto>
+ reserve(QueryId queryId, NodeResourceRequest request) {
+
+ List<AllocationResourceProto> reservedResources;
+ NodeResource capacity = new NodeResource(request.getCapacity());
+ if (!NodeResources.fitsIn(capacity, getClusterResource())) {
+ return Lists.newArrayList();
+ }
+
+ LinkedList<Integer> workers = new LinkedList<Integer>();
+
+ if (request.getCandidateNodesCount() > 0) {
+ workers.addAll(request.getCandidateNodesList());
+ Collections.shuffle(workers);
+ }
+
+ int requiredContainers = request.getNumContainers();
+ // reserve resource from candidate workers for locality
+ reservedResources = reserveClusterResource(workers, capacity, requiredContainers);
+
+ // reserve resource in random workers
+ if (reservedResources.size() < requiredContainers) {
+ LinkedList<Integer> randomNodes = new LinkedList<Integer>(getRMContext().getNodes().keySet());
+ Collections.shuffle(randomNodes);
+
+ reservedResources.addAll(reserveClusterResource(
+ randomNodes, capacity, requiredContainers - reservedResources.size()));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Request: " + request.getCapacity() + ", containerNum:" + request.getNumContainers()
+ + "Current cluster resource: " + getClusterResource());
+ }
+ return reservedResources;
+ }
+
+ private List<AllocationResourceProto> reserveClusterResource(List<Integer> workers,
+ NodeResource capacity, int requiredNum) {
+
+ List<AllocationResourceProto> reservedResources = Lists.newArrayList();
+ AllocationResourceProto.Builder resourceBuilder = AllocationResourceProto.newBuilder();
+ int allocatedResources = 0;
+
+ while (workers.size() > 0) {
+ Iterator<Integer> iter = workers.iterator();
+ while (iter.hasNext()) {
+
+ int workerId = iter.next();
+ NodeStatus nodeStatus = getRMContext().getNodes().get(workerId);
+ if (nodeStatus == null) {
+ iter.remove();
+ LOG.warn("Can't find the node. id :" + workerId);
+ continue;
+ } else {
+ if (NodeResources.fitsIn(capacity, nodeStatus.getAvailableResource())) {
+ NodeResources.subtractFrom(getClusterResource(), capacity);
+ NodeResources.subtractFrom(nodeStatus.getAvailableResource(), capacity);
+ allocatedResources++;
+ resourceBuilder.setResource(capacity.getProto());
+ resourceBuilder.setWorkerId(workerId);
+ reservedResources.add(resourceBuilder.build());
+ } else {
+ // remove unavailable nodeStatus;
+ iter.remove();
+ }
+ }
+
+ if (allocatedResources >= requiredNum) {
+ return reservedResources;
+ }
+ }
+ }
+ return reservedResources;
+ }
+
+
+ @Override
+ public void handle(SchedulerEvent event) {
+ switch (event.getType()) {
+ case RESOURCE_RESERVE:
+ //TODO should consider request priority
+ reserveResource(TUtil.checkTypeAndGet(event, ResourceReserveSchedulerEvent.class));
+ break;
+ case RESOURCE_UPDATE:
+ updateResource();
+ break;
+ default:
+ break;
+
+ }
+ }
+
+ /**
+ * This is an asynchronous call. You should use a callback to get reserved resource containers.
+ */
+ protected void reserveResource(ResourceReserveSchedulerEvent schedulerEvent) {
+ List<AllocationResourceProto> resources =
+ reserve(new QueryId(schedulerEvent.getRequest().getQueryId()), schedulerEvent.getRequest());
+
+ NodeResourceResponse.Builder response = NodeResourceResponse.newBuilder();
+ response.setQueryId(schedulerEvent.getRequest().getQueryId());
+ schedulerEvent.getCallBack().run(response.addAllResource(resources).build());
+ }
+
+ /**
+ * Submit a query to scheduler
+ */
+ public void submitQuery(QuerySchedulingInfo schedulingInfo) {
+ queryQueue.add(schedulingInfo);
+ pendingQueryMap.put(schedulingInfo.getQueryId(), schedulingInfo);
+ }
+
+ protected boolean startQuery(QueryId queryId, AllocationResourceProto allocation) {
+ return masterContext.getQueryJobManager().startQueryJob(queryId, allocation);
+ }
+
+ public void stopQuery(QueryId queryId) {
+ if(pendingQueryMap.containsKey(queryId)){
+ queryQueue.remove(pendingQueryMap.remove(queryId));
+ }
+ assignedQueryMasterMap.remove(queryId);
+ }
+
+ public BlockingQueue<QuerySchedulingInfo> getQueryQueue() {
+ return queryQueue;
+ }
+
+ private NodeStatus getWorker(int workerId) {
+ return rmContext.getNodes().get(workerId);
+ }
+
+ protected TajoRMContext getRMContext() {
+ return rmContext;
+ }
+
+ public WorkerConnectionInfo getQueryMaster(QueryId queryId) {
+ if (assignedQueryMasterMap.containsKey(queryId)) {
+ return rmContext.getNodes().get(assignedQueryMasterMap.get(queryId)).getConnectionInfo();
+ }
+ return null;
+ }
+
+ protected QueryInfo getQueryInfo(QueryId queryId) {
+ return masterContext.getQueryJobManager().getQueryInProgress(queryId).getQueryInfo();
+ }
+
+ private final class QueryProcessor implements Runnable {
+ @Override
+ public void run() {
+
+ QuerySchedulingInfo query;
+
+ while (!isStopped && !Thread.currentThread().isInterrupted()) {
+ try {
+ query = queryQueue.take();
+ } catch (InterruptedException e) {
+ LOG.warn(e.getMessage(), e);
+ break;
+ }
+ //TODO get by assigned queue
+ int maxAvailable = getResourceCalculator().computeAvailableContainers(
+ getMaximumResourceCapability(), getQMMinimumResourceCapability());
+
+ // check maximum parallel running QM. allow 50% parallel running
+ if (assignedQueryMasterMap.size() >= Math.floor(maxAvailable * MAXIMUM_RUNNING_QM_RATE)) {
+ queryQueue.add(query);
+ synchronized (this) {
+ try {
+ this.wait(1000);
+ } catch (InterruptedException e) {
+ if(!isStopped) {
+ LOG.fatal(e.getMessage(), e);
+ return;
+ }
+ }
+ }
+ } else {
+ QueryInfo queryInfo = getQueryInfo(query.getQueryId());
+ List<AllocationResourceProto> allocation = reserve(query.getQueryId(), createQMResourceRequest(queryInfo));
+
+ if(allocation.size() == 0) {
+ queryQueue.add(query);
+ LOG.info("No Available Resources for QueryMaster :" + queryInfo.getQueryId() + "," + queryInfo);
+
+ synchronized (this) {
+ try {
+ this.wait(100);
+ } catch (InterruptedException e) {
+ LOG.fatal(e);
+ }
+ }
+ } else {
+ try {
+ //if QM resource can't be allocated to a node, it should retry
+ boolean started = startQuery(query.getQueryId(), allocation.get(0));
+ if(!started) {
+ queryQueue.put(query);
+ } else {
+ assignedQueryMasterMap.put(query.getQueryId(), allocation.get(0).getWorkerId());
+ }
+ } catch (Throwable t) {
+ LOG.fatal("Exception during query startup:", t);
+ masterContext.getQueryJobManager().stopQuery(query.getQueryId());
+ }
+ }
+ }
+ LOG.info("Running Queries: " + assignedQueryMasterMap.size());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/TajoResourceScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/TajoResourceScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/TajoResourceScheduler.java
new file mode 100644
index 0000000..c7c37c4
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/TajoResourceScheduler.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler;
+
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tajo.QueryId;
+import org.apache.tajo.ResourceProtos.NodeResourceRequest;
+import org.apache.tajo.ResourceProtos.AllocationResourceProto;
+import org.apache.tajo.master.scheduler.event.SchedulerEvent;
+import org.apache.tajo.resource.NodeResource;
+
+import java.util.List;
+
+/**
+ * This interface is used by scheduler for allocating of resources.
+ */
+public interface TajoResourceScheduler extends EventHandler<SchedulerEvent> {
+
+ /**
+ * Get the whole resource capacity of the cluster.
+ * @return the whole resource capacity of the cluster.
+ */
+
+ NodeResource getClusterResource();
+
+ /**
+ * Get minimum allocatable {@link NodeResource}.
+ * @return minimum allocatable resource
+ */
+ NodeResource getMinimumResourceCapability();
+
+ /**
+ * Get minimum allocatable {@link NodeResource} of QueryMaster.
+ * @return minimum allocatable resource
+ */
+ NodeResource getQMMinimumResourceCapability();
+
+ /**
+ * Get maximum allocatable {@link NodeResource}.
+ * @return maximum allocatable resource
+ */
+ NodeResource getMaximumResourceCapability();
+
+ /**
+ * Get the number of nodes available in the cluster.
+ * @return the number of available nodes.
+ */
+ int getNumClusterNodes();
+
+ /**
+ * Get reservation resource. The cluster resource is updated by TajoResourceTracker
+ * Request one or more resource containers. You can set the number of containers and resource capabilities,
+ * such as memory, CPU cores, and disk slots.
+ * @return the number of reserved resources.
+ */
+ List<AllocationResourceProto>
+ reserve(QueryId queryId, NodeResourceRequest ask);
+
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/ResourceReserveSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/ResourceReserveSchedulerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/ResourceReserveSchedulerEvent.java
new file mode 100644
index 0000000..47ee53b
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/ResourceReserveSchedulerEvent.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler.event;
+
+import com.google.protobuf.RpcCallback;
+import static org.apache.tajo.ResourceProtos.NodeResourceRequest;
+import static org.apache.tajo.ResourceProtos.NodeResourceResponse;
+
+public class ResourceReserveSchedulerEvent extends SchedulerEvent {
+
+ private NodeResourceRequest request;
+
+ private RpcCallback<NodeResourceResponse> callBack;
+
+ public ResourceReserveSchedulerEvent(NodeResourceRequest request,
+ RpcCallback<NodeResourceResponse> callback) {
+ super(SchedulerEventType.RESOURCE_RESERVE);
+ this.request = request;
+ this.callBack = callback;
+ }
+
+ public NodeResourceRequest getRequest() {
+ return request;
+ }
+
+ public RpcCallback<NodeResourceResponse> getCallBack() {
+ return callBack;
+ }
+}
[09/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index bc0d212..0ace53a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@ -41,8 +41,7 @@ import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.planner.physical.PhysicalExec;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.TajoMaster.MasterContext;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
+import org.apache.tajo.master.rm.NodeStatus;
import org.apache.tajo.plan.InvalidQueryException;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
@@ -50,6 +49,8 @@ import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.IndexScanNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.ScanNode;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.resource.NodeResources;
import org.apache.tajo.session.InvalidSessionException;
import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder;
@@ -420,40 +421,35 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
return tuples;
}
- private Tuple getQueryMasterTuple(Schema outSchema, Worker aWorker) {
+ private Tuple getQueryMasterTuple(Schema outSchema, NodeStatus aNodeStatus) {
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
- WorkerResource aResource = aWorker.getResource();
-
+
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("host".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getConnectionInfo() != null && aWorker.getConnectionInfo().getHost() != null) {
- aTuple.put(fieldId, DatumFactory.createText(aWorker.getConnectionInfo().getHost()));
+ if (aNodeStatus.getConnectionInfo() != null && aNodeStatus.getConnectionInfo().getHost() != null) {
+ aTuple.put(fieldId, DatumFactory.createText(aNodeStatus.getConnectionInfo().getHost()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("port".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getConnectionInfo() != null) {
- aTuple.put(fieldId, DatumFactory.createInt4(aWorker.getConnectionInfo().getQueryMasterPort()));
+ if (aNodeStatus.getConnectionInfo() != null) {
+ aTuple.put(fieldId, DatumFactory.createInt4(aNodeStatus.getConnectionInfo().getQueryMasterPort()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("type".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText("QueryMaster"));
} else if ("status".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(aWorker.getState().toString()));
- } else if ("RUNNING".equalsIgnoreCase(aWorker.getState().toString())) {
+ aTuple.put(fieldId, DatumFactory.createText(aNodeStatus.getState().toString()));
+ } else if ("RUNNING".equalsIgnoreCase(aNodeStatus.getState().toString())) {
if ("running_tasks".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(aResource.getNumQueryMasterTasks()));
- } else if ("free_heap".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getFreeHeap()));
- } else if ("max_heap".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getMaxHeap()));
+ aTuple.put(fieldId, DatumFactory.createInt4(aNodeStatus.getNumRunningQueryMaster()));
} else if ("last_heartbeat_ts".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getLastHeartbeatTime() > 0) {
- aTuple.put(fieldId, DatumFactory.createTimestmpDatumWithJavaMillis(aWorker.getLastHeartbeatTime()));
+ if (aNodeStatus.getLastHeartbeatTime() > 0) {
+ aTuple.put(fieldId, DatumFactory.createTimestmpDatumWithJavaMillis(aNodeStatus.getLastHeartbeatTime()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
@@ -466,50 +462,48 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
return aTuple;
}
- private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) {
+ private Tuple getWorkerTuple(Schema outSchema, NodeStatus aNodeStatus) {
List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
- WorkerResource aResource = aWorker.getResource();
+
+ NodeResource total = aNodeStatus.getTotalResourceCapability();
+ NodeResource used = NodeResources.subtract(total, aNodeStatus.getAvailableResource());
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
if ("host".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getConnectionInfo() != null && aWorker.getConnectionInfo().getHost() != null) {
- aTuple.put(fieldId, DatumFactory.createText(aWorker.getConnectionInfo().getHost()));
+ if (aNodeStatus.getConnectionInfo() != null && aNodeStatus.getConnectionInfo().getHost() != null) {
+ aTuple.put(fieldId, DatumFactory.createText(aNodeStatus.getConnectionInfo().getHost()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("port".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getConnectionInfo() != null) {
- aTuple.put(fieldId, DatumFactory.createInt4(aWorker.getConnectionInfo().getPeerRpcPort()));
+ if (aNodeStatus.getConnectionInfo() != null) {
+ aTuple.put(fieldId, DatumFactory.createInt4(aNodeStatus.getConnectionInfo().getPeerRpcPort()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
} else if ("type".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText("Worker"));
} else if ("status".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(aWorker.getState().toString()));
- } else if ("RUNNING".equalsIgnoreCase(aWorker.getState().toString())) {
+ aTuple.put(fieldId, DatumFactory.createText(aNodeStatus.getState().toString()));
+ } else if ("RUNNING".equalsIgnoreCase(aNodeStatus.getState().toString())) {
if ("total_cpu".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(aResource.getCpuCoreSlots()));
+ aTuple.put(fieldId, DatumFactory.createInt4(total.getVirtualCores()));
} else if ("used_mem".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getUsedMemoryMB()*1048576l));
+ aTuple.put(fieldId, DatumFactory.createInt8(used.getMemory() * 1048576l));
} else if ("total_mem".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getMemoryMB()*1048576l));
- } else if ("free_heap".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getFreeHeap()));
- } else if ("max_heap".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt8(aResource.getMaxHeap()));
- } else if ("used_diskslots".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createFloat4(aResource.getUsedDiskSlots()));
- } else if ("total_diskslots".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createFloat4(aResource.getDiskSlots()));
+ aTuple.put(fieldId, DatumFactory.createInt8(total.getMemory() * 1048576l));
+ } else if ("used_disk".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createInt4(used.getDisks()));
+ } else if ("total_disk".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createInt4(total.getDisks()));
} else if ("running_tasks".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(aResource.getNumRunningTasks()));
+ aTuple.put(fieldId, DatumFactory.createInt4(aNodeStatus.getNumRunningTasks()));
} else if ("last_heartbeat_ts".equalsIgnoreCase(column.getSimpleName())) {
- if (aWorker.getLastHeartbeatTime() > 0) {
- aTuple.put(fieldId, DatumFactory.createTimestmpDatumWithJavaMillis(aWorker.getLastHeartbeatTime()));
+ if (aNodeStatus.getLastHeartbeatTime() > 0) {
+ aTuple.put(fieldId, DatumFactory.createTimestmpDatumWithJavaMillis(aNodeStatus.getLastHeartbeatTime()));
} else {
aTuple.put(fieldId, DatumFactory.createNullDatum());
}
@@ -524,23 +518,23 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private List<Tuple> getClusterInfo(Schema outSchema) {
- Map<Integer, Worker> workerMap = masterContext.getResourceManager().getWorkers();
+ Map<Integer, NodeStatus> workerMap = masterContext.getResourceManager().getNodes();
List<Tuple> tuples;
- List<Worker> queryMasterList = new ArrayList<Worker>();
- List<Worker> workerList = new ArrayList<Worker>();
+ List<NodeStatus> queryMasterList = new ArrayList<NodeStatus>();
+ List<NodeStatus> nodeStatusList = new ArrayList<NodeStatus>();
- for (Worker aWorker: workerMap.values()) {
- queryMasterList.add(aWorker);
- workerList.add(aWorker);
+ for (NodeStatus aNodeStatus : workerMap.values()) {
+ queryMasterList.add(aNodeStatus);
+ nodeStatusList.add(aNodeStatus);
}
- tuples = new ArrayList<Tuple>(queryMasterList.size() + workerList.size());
- for (Worker queryMaster: queryMasterList) {
+ tuples = new ArrayList<Tuple>(queryMasterList.size() + nodeStatusList.size());
+ for (NodeStatus queryMaster: queryMasterList) {
tuples.add(getQueryMasterTuple(outSchema, queryMaster));
}
- for (Worker worker: workerList) {
- tuples.add(getWorkerTuple(outSchema, worker));
+ for (NodeStatus nodeStatus : nodeStatusList) {
+ tuples.add(getWorkerTuple(outSchema, nodeStatus));
}
return tuples;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEvent.java
new file mode 100644
index 0000000..430dc86
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+/**
+ * NodeEvent describes all kinds of events which sent to {@link NodeStatus}.
+ */
+public class NodeEvent extends AbstractEvent<NodeEventType> {
+ private final int workerId;
+
+ public NodeEvent(int workerId, NodeEventType nodeEventType) {
+ super(nodeEventType);
+ this.workerId = workerId;
+ }
+
+ public int getWorkerId() {
+ return workerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEventType.java
new file mode 100644
index 0000000..64de5d4
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeEventType.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+public enum NodeEventType {
+
+ /** Source : {@link TajoResourceTracker}, Destination: {@link NodeStatus} */
+ STARTED,
+ STATE_UPDATE,
+ RECONNECTED,
+
+ /** Source : {@link NodeLivelinessMonitor}, Destination: {@link NodeStatus} */
+ EXPIRE
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeLivelinessMonitor.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeLivelinessMonitor.java
new file mode 100644
index 0000000..55a24fc
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeLivelinessMonitor.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.AbstractLivelinessMonitor;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tajo.conf.TajoConf;
+
+/**
+ * It periodically checks the latest heartbeat time of {@link NodeStatus}.
+ * If the latest heartbeat time is expired, it produces EXPIRE event to a corresponding {@link NodeStatus}.
+ */
+public class NodeLivelinessMonitor extends AbstractLivelinessMonitor<Integer> {
+
+ private EventHandler dispatcher;
+
+ public NodeLivelinessMonitor(Dispatcher d) {
+ super(NodeLivelinessMonitor.class.getSimpleName(), new SystemClock());
+ this.dispatcher = d.getEventHandler();
+ }
+
+ public void serviceInit(Configuration conf) throws Exception {
+ Preconditions.checkArgument(conf instanceof TajoConf);
+ TajoConf systemConf = (TajoConf) conf;
+ // milliseconds
+ int expireIntvl = systemConf.getIntVar(TajoConf.ConfVars.RESOURCE_TRACKER_HEARTBEAT_TIMEOUT) * 1000;
+ setExpireInterval(expireIntvl);
+ setMonitorInterval(expireIntvl/3);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void expire(Integer id) {
+ dispatcher.handle(new NodeEvent(id, NodeEventType.EXPIRE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeReconnectEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeReconnectEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeReconnectEvent.java
new file mode 100644
index 0000000..e9ba077
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeReconnectEvent.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+/**
+ * {@link TajoResourceTracker} produces this event, and it's destination is {@link NodeStatus}.
+ * This event occurs only when an inactive nodeStatus sends a ping again.
+ */
+public class NodeReconnectEvent extends NodeEvent {
+ private final NodeStatus nodeStatus;
+ public NodeReconnectEvent(int workerId, NodeStatus nodeStatus) {
+ super(workerId, NodeEventType.RECONNECTED);
+ this.nodeStatus = nodeStatus;
+ }
+
+ public NodeStatus getNodeStatus() {
+ return nodeStatus;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeState.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeState.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeState.java
new file mode 100644
index 0000000..89f601d
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeState.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+/**
+ * It presents the states of {@link NodeStatus}.
+ */
+public enum NodeState {
+ /** New node */
+ NEW,
+
+ /** Running node */
+ RUNNING,
+
+ /** Node is unhealthy */
+ UNHEALTHY,
+
+ /** Node is out of service */
+ DECOMMISSIONED,
+
+ /** Node has not sent a heartbeat for some configured time threshold */
+ LOST;
+
+ @SuppressWarnings("unused")
+ public boolean isUnusable() {
+ return (this == UNHEALTHY || this == DECOMMISSIONED || this == LOST);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatus.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatus.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatus.java
new file mode 100644
index 0000000..63e4d52
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatus.java
@@ -0,0 +1,295 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+import io.netty.util.internal.PlatformDependent;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.state.*;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.util.TUtil;
+
+import java.util.EnumSet;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+/**
+ * It contains resource and various information for a node.
+ */
+public class NodeStatus implements EventHandler<NodeEvent>, Comparable<NodeStatus> {
+ /** class logger */
+ private static final Log LOG = LogFactory.getLog(NodeStatus.class);
+
+ /** context of {@link TajoResourceManager} */
+ private final TajoRMContext rmContext;
+
+ /** last heartbeat time */
+ private volatile long lastHeartbeatTime;
+
+ @SuppressWarnings("unused")
+ private volatile int numRunningTasks;
+
+ @SuppressWarnings("unused")
+ private volatile int numRunningQueryMaster;
+
+ private static AtomicLongFieldUpdater HEARTBEAT_TIME_UPDATER;
+ private static AtomicIntegerFieldUpdater RUNNING_TASK_UPDATER;
+ private static AtomicIntegerFieldUpdater RUNNING_QM_UPDATER;
+
+ static {
+ HEARTBEAT_TIME_UPDATER = PlatformDependent.newAtomicLongFieldUpdater(NodeStatus.class, "lastHeartbeatTime");
+ if (HEARTBEAT_TIME_UPDATER == null) {
+ HEARTBEAT_TIME_UPDATER = AtomicLongFieldUpdater.newUpdater(NodeStatus.class, "lastHeartbeatTime");
+ RUNNING_TASK_UPDATER = AtomicIntegerFieldUpdater.newUpdater(NodeStatus.class, "numRunningTasks");
+ RUNNING_QM_UPDATER = AtomicIntegerFieldUpdater.newUpdater(NodeStatus.class, "numRunningQueryMaster");
+ } else {
+ RUNNING_TASK_UPDATER = PlatformDependent.newAtomicIntegerFieldUpdater(NodeStatus.class, "numRunningTasks");
+ RUNNING_QM_UPDATER = PlatformDependent.newAtomicIntegerFieldUpdater(NodeStatus.class, "numRunningQueryMaster");
+ }
+ }
+
+ /** Available resources on the node. */
+ private final NodeResource availableResource;
+
+ /** Total resources on the node. */
+ private final NodeResource totalResourceCapability;
+
+ /** Node connection information */
+ private WorkerConnectionInfo connectionInfo;
+
+ private static final ReconnectNodeTransition RECONNECT_NODE_TRANSITION = new ReconnectNodeTransition();
+ private static final StatusUpdateTransition STATUS_UPDATE_TRANSITION = new StatusUpdateTransition();
+
+ private static final StateMachineFactory<NodeStatus,
+ NodeState,
+ NodeEventType,
+ NodeEvent> stateMachineFactory
+ = new StateMachineFactory<NodeStatus,
+ NodeState,
+ NodeEventType,
+ NodeEvent>(NodeState.NEW)
+
+ // Transition from NEW
+ .addTransition(NodeState.NEW, NodeState.RUNNING,
+ NodeEventType.STARTED,
+ new AddNodeTransition())
+
+ // Transition from RUNNING
+ .addTransition(NodeState.RUNNING, EnumSet.of(NodeState.RUNNING, NodeState.UNHEALTHY),
+ NodeEventType.STATE_UPDATE,
+ STATUS_UPDATE_TRANSITION)
+ .addTransition(NodeState.RUNNING, NodeState.LOST,
+ NodeEventType.EXPIRE,
+ new DeactivateNodeTransition(NodeState.LOST))
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
+ NodeEventType.RECONNECTED,
+ RECONNECT_NODE_TRANSITION)
+
+ // Transitions from UNHEALTHY state
+ .addTransition(NodeState.UNHEALTHY, EnumSet.of(NodeState.RUNNING, NodeState.UNHEALTHY),
+ NodeEventType.STATE_UPDATE,
+ STATUS_UPDATE_TRANSITION)
+ .addTransition(NodeState.UNHEALTHY, NodeState.LOST,
+ NodeEventType.EXPIRE,
+ new DeactivateNodeTransition(NodeState.LOST))
+ .addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
+ NodeEventType.RECONNECTED,
+ RECONNECT_NODE_TRANSITION);
+
+ private final StateMachine<NodeState, NodeEventType, NodeEvent> stateMachine =
+ stateMachineFactory.make(this, NodeState.NEW);
+
+ public NodeStatus(TajoRMContext rmContext, NodeResource totalResourceCapability, WorkerConnectionInfo connectionInfo) {
+ this.rmContext = rmContext;
+
+ this.connectionInfo = connectionInfo;
+ this.lastHeartbeatTime = System.currentTimeMillis();
+ this.totalResourceCapability = totalResourceCapability;
+ this.availableResource = NodeResources.clone(totalResourceCapability);
+ }
+
+ public int getWorkerId() {
+ return connectionInfo.getId();
+ }
+
+ public WorkerConnectionInfo getConnectionInfo() {
+ return connectionInfo;
+ }
+
+ public void setLastHeartbeatTime(long lastHeartbeatTime) {
+ HEARTBEAT_TIME_UPDATER.lazySet(this, lastHeartbeatTime);
+ }
+
+ public void setNumRunningQueryMaster(int numRunningQueryMaster) {
+ RUNNING_QM_UPDATER.lazySet(this, numRunningQueryMaster);
+ }
+
+ public int getNumRunningQueryMaster() {
+ return numRunningQueryMaster;
+ }
+
+ public void setNumRunningTasks(int numRunningTasks) {
+ RUNNING_TASK_UPDATER.lazySet(this, numRunningTasks);
+ }
+
+ public int getNumRunningTasks() {
+ return numRunningTasks;
+ }
+
+ public long getLastHeartbeatTime() {
+ return this.lastHeartbeatTime;
+ }
+
+ /**
+ *
+ * @return the current state of node
+ */
+ public NodeState getState() {
+ return this.stateMachine.getCurrentState();
+ }
+
+ /**
+ * Get current resources on the node.
+ *
+ * @return current resources on the node.
+ */
+ public NodeResource getAvailableResource() {
+ return this.availableResource;
+ }
+
+ /**
+ * Get total resources on the node.
+ *
+ * @return total resources on the node.
+ */
+ public NodeResource getTotalResourceCapability() {
+ return totalResourceCapability;
+ }
+
+ @Override
+ public int compareTo(NodeStatus o) {
+ if(o == null) {
+ return 1;
+ }
+ return connectionInfo.compareTo(o.connectionInfo);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ NodeStatus nodeStatus = (NodeStatus) o;
+
+ if (connectionInfo != null ? !connectionInfo.equals(nodeStatus.connectionInfo) : nodeStatus.connectionInfo != null)
+ return false;
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 0;
+ result = 31 * result + (connectionInfo != null ? connectionInfo.hashCode() : 0);
+ return result;
+ }
+
+ public static class AddNodeTransition implements SingleArcTransition<NodeStatus, NodeEvent> {
+ @Override
+ public void transition(NodeStatus nodeStatus, NodeEvent nodeEvent) {
+
+ nodeStatus.rmContext.getQueryMasterWorker().add(nodeStatus.getWorkerId());
+ LOG.info("Node with " + nodeStatus.getTotalResourceCapability() + " is joined to Tajo cluster");
+ }
+ }
+
+ public static class StatusUpdateTransition implements
+ MultipleArcTransition<NodeStatus, NodeEvent, NodeState> {
+
+ @Override
+ public NodeState transition(NodeStatus nodeStatus, NodeEvent event) {
+
+ NodeStatusEvent statusEvent = TUtil.checkTypeAndGet(event, NodeStatusEvent.class);
+ nodeStatus.updateStatus(statusEvent);
+
+ return NodeState.RUNNING;
+ }
+ }
+
+ private void updateStatus(NodeStatusEvent statusEvent) {
+ setLastHeartbeatTime(System.currentTimeMillis());
+ setNumRunningTasks(statusEvent.getRunningTaskNum());
+ setNumRunningQueryMaster(statusEvent.getRunningQMNum());
+ NodeResources.update(availableResource, statusEvent.getAvailableResource());
+
+ if(statusEvent.getTotalResource() != null) {
+ NodeResources.update(totalResourceCapability, statusEvent.getTotalResource());
+ }
+ }
+
+ public static class DeactivateNodeTransition implements SingleArcTransition<NodeStatus, NodeEvent> {
+ private final NodeState finalState;
+
+ public DeactivateNodeTransition(NodeState finalState) {
+ this.finalState = finalState;
+ }
+
+ @Override
+ public void transition(NodeStatus nodeStatus, NodeEvent nodeEvent) {
+
+ nodeStatus.rmContext.getNodes().remove(nodeStatus.getWorkerId());
+ LOG.info("Deactivating Node " + nodeStatus.getWorkerId() + " as it is now " + finalState);
+ nodeStatus.rmContext.getInactiveNodes().putIfAbsent(nodeStatus.getWorkerId(), nodeStatus);
+ }
+ }
+
+ public static class ReconnectNodeTransition implements SingleArcTransition<NodeStatus, NodeEvent> {
+
+ @Override
+ public void transition(NodeStatus nodeStatus, NodeEvent nodeEvent) {
+
+ NodeReconnectEvent castedEvent = TUtil.checkTypeAndGet(nodeEvent, NodeReconnectEvent.class);
+ NodeStatus newNodeStatus = castedEvent.getNodeStatus();
+ nodeStatus.rmContext.getNodes().put(castedEvent.getWorkerId(), newNodeStatus);
+ nodeStatus.rmContext.getDispatcher().getEventHandler().handle(
+ new NodeEvent(nodeStatus.getWorkerId(), NodeEventType.STARTED));
+ }
+ }
+
+ @Override
+ public void handle(NodeEvent event) {
+ LOG.debug("Processing " + event.getWorkerId() + " of type " + event.getType());
+ NodeState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitonException e) {
+ LOG.error("Can't handle this event at current state"
+ + ", eventType:" + event.getType().name()
+ + ", oldState:" + oldState.name()
+ + ", nextState:" + getState().name()
+ , e);
+ LOG.error("Invalid event " + event.getType() + " on NodeStatus " + getWorkerId());
+ }
+ if (oldState != getState()) {
+ LOG.info(getWorkerId() + " Node Transitioned from " + oldState + " to " + getState());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatusEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatusEvent.java
new file mode 100644
index 0000000..db2f167
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/NodeStatusEvent.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+import org.apache.tajo.annotation.Nullable;
+import org.apache.tajo.resource.NodeResource;
+
+/**
+ * {@link TajoResourceTracker} produces this event, and its destination is
+ * {@link NodeStatus.StatusUpdateTransition} of {@link NodeStatus}.
+ */
+public class NodeStatusEvent extends NodeEvent {
+ private final int runningTaskNum;
+ private final int runningQMNum;
+ private final NodeResource available;
+ private final NodeResource total;
+
+ public NodeStatusEvent(int workerId, int runningTaskNum, int runningQMNum,
+ NodeResource available, @Nullable NodeResource total) {
+ super(workerId, NodeEventType.STATE_UPDATE);
+ this.runningTaskNum = runningTaskNum;
+ this.runningQMNum = runningQMNum;
+ this.available = available;
+ this.total = total;
+ }
+
+ public int getRunningTaskNum() {
+ return runningTaskNum;
+ }
+
+ public int getRunningQMNum() {
+ return runningQMNum;
+ }
+
+ public NodeResource getAvailableResource() {
+ return available;
+ }
+
+ public NodeResource getTotalResource() {
+ return total;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
index bb8cc12..9dfae7e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
@@ -20,8 +20,6 @@ package org.apache.tajo.master.rm;
import com.google.common.collect.Maps;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.tajo.QueryId;
-import org.apache.tajo.ipc.ContainerProtocol;
import java.util.Collections;
import java.util.Set;
@@ -29,27 +27,21 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
/**
- * It's a worker resource manager context. It contains all context data about TajoWorkerResourceManager.
+ * It's a worker resource manager context. It contains all context data about TajoResourceManager.
*/
public class TajoRMContext {
final Dispatcher rmDispatcher;
- /** map between workerIds and running workers */
- private final ConcurrentMap<Integer, Worker> workers = Maps.newConcurrentMap();
+ /** map between workerIds and running nodes */
+ private final ConcurrentMap<Integer, NodeStatus> nodes = Maps.newConcurrentMap();
- /** map between workerIds and inactive workers */
- private final ConcurrentMap<Integer, Worker> inactiveWorkers = Maps.newConcurrentMap();
-
- /** map between queryIds and query master ContainerId */
- private final ConcurrentMap<QueryId, ContainerProtocol.TajoContainerIdProto> qmContainerMap = Maps
- .newConcurrentMap();
+ /** map between workerIds and inactive nodes */
+ private final ConcurrentMap<Integer, NodeStatus> inactiveNodes = Maps.newConcurrentMap();
private final Set<Integer> liveQueryMasterWorkerResources =
Collections.newSetFromMap(new ConcurrentHashMap<Integer, Boolean>());
- private final Set<QueryId> stoppedQueryIds =
- Collections.newSetFromMap(new ConcurrentHashMap<QueryId, Boolean>());
public TajoRMContext(Dispatcher dispatcher) {
this.rmDispatcher = dispatcher;
@@ -60,32 +52,20 @@ public class TajoRMContext {
}
/**
- * @return The Map for active workers
+ * @return The Map for active nodes
*/
- public ConcurrentMap<Integer, Worker> getWorkers() {
- return workers;
+ public ConcurrentMap<Integer, NodeStatus> getNodes() {
+ return nodes;
}
/**
- * @return The Map for inactive workers
+ * @return The Map for inactive nodes
*/
- public ConcurrentMap<Integer, Worker> getInactiveWorkers() {
- return inactiveWorkers;
- }
-
- /**
- *
- * @return The Map for query master containers
- */
- public ConcurrentMap<QueryId, ContainerProtocol.TajoContainerIdProto> getQueryMasterContainer() {
- return qmContainerMap;
+ public ConcurrentMap<Integer, NodeStatus> getInactiveNodes() {
+ return inactiveNodes;
}
public Set<Integer> getQueryMasterWorker() {
return liveQueryMasterWorkerResources;
}
-
- public Set<QueryId> getStoppedQueryIds() {
- return stoppedQueryIds;
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceManager.java
new file mode 100644
index 0000000..c91938a
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceManager.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.rm;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.master.scheduler.AbstractQueryScheduler;
+import org.apache.tajo.master.scheduler.QuerySchedulingInfo;
+import org.apache.tajo.master.scheduler.event.SchedulerEventType;
+import org.apache.tajo.util.TUtil;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * It manages all resources of tajo workers.
+ */
+public class TajoResourceManager extends CompositeService {
+ /** class logger */
+ private static final Log LOG = LogFactory.getLog(TajoResourceManager.class);
+
+ protected static final Map<String, Class<? extends AbstractQueryScheduler>> SCHEDULER_CLASS_CACHE = Maps.newHashMap();
+
+ private TajoMaster.MasterContext masterContext;
+
+ private TajoRMContext rmContext;
+
+ private String queryIdSeed;
+
+ /**
+ * Node Liveliness monitor
+ */
+ private NodeLivelinessMonitor nodeLivelinessMonitor;
+
+ private TajoConf systemConf;
+ private AbstractQueryScheduler scheduler;
+
+ /** It receives status messages from workers and their resources. */
+ private TajoResourceTracker resourceTracker;
+
+ public TajoResourceManager(TajoMaster.MasterContext masterContext) {
+ super(TajoResourceManager.class.getSimpleName());
+ this.masterContext = masterContext;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+
+ AsyncDispatcher dispatcher = new AsyncDispatcher();
+ addIfService(dispatcher);
+
+ rmContext = new TajoRMContext(dispatcher);
+
+ this.queryIdSeed = String.valueOf(System.currentTimeMillis());
+
+ this.nodeLivelinessMonitor = new NodeLivelinessMonitor(this.rmContext.getDispatcher());
+ addIfService(this.nodeLivelinessMonitor);
+
+ // Register event handler for Workers
+ rmContext.getDispatcher().register(NodeEventType.class, new WorkerEventDispatcher(rmContext));
+
+ resourceTracker = new TajoResourceTracker(this, nodeLivelinessMonitor);
+ addIfService(resourceTracker);
+
+ String schedulerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_SCHEDULER_CLASS);
+ scheduler = loadScheduler(schedulerClassName);
+ LOG.info("Loaded resource scheduler : " + scheduler.getClass());
+ addIfService(scheduler);
+ rmContext.getDispatcher().register(SchedulerEventType.class, scheduler);
+
+ super.serviceInit(systemConf);
+ }
+
+ protected synchronized AbstractQueryScheduler loadScheduler(String schedulerClassName) throws Exception {
+ Class<? extends AbstractQueryScheduler> schedulerClass;
+ if (SCHEDULER_CLASS_CACHE.containsKey(schedulerClassName)) {
+ schedulerClass = SCHEDULER_CLASS_CACHE.get(schedulerClassName);
+ } else {
+ schedulerClass = (Class<? extends AbstractQueryScheduler>) Class.forName(schedulerClassName);
+ SCHEDULER_CLASS_CACHE.put(schedulerClassName, schedulerClass);
+ }
+
+ Constructor<? extends AbstractQueryScheduler>
+ constructor = schedulerClass.getDeclaredConstructor(new Class[]{TajoMaster.MasterContext.class});
+ constructor.setAccessible(true);
+
+ return constructor.newInstance(new Object[]{masterContext});
+ }
+
+ @InterfaceAudience.Private
+ public static final class WorkerEventDispatcher implements EventHandler<NodeEvent> {
+
+ private final TajoRMContext rmContext;
+
+ public WorkerEventDispatcher(TajoRMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void handle(NodeEvent event) {
+ int workerId = event.getWorkerId();
+ NodeStatus node = this.rmContext.getNodes().get(workerId);
+ if (node != null) {
+ try {
+ node.handle(event);
+ } catch (Throwable t) {
+ LOG.error("Error in handling event type " + event.getType() + " for node " + workerId, t);
+ }
+ }
+ }
+ }
+
+
+ public Map<Integer, NodeStatus> getNodes() {
+ return ImmutableMap.copyOf(rmContext.getNodes());
+ }
+
+ public Map<Integer, NodeStatus> getInactiveNodes() {
+ return ImmutableMap.copyOf(rmContext.getInactiveNodes());
+ }
+
+ public Collection<Integer> getQueryMasters() {
+ return Collections.unmodifiableSet(rmContext.getQueryMasterWorker());
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ /**
+ *
+ * @return The prefix of queryId. It is generated when a TajoMaster starts up.
+ */
+ public String getSeedQueryId() throws IOException {
+ return queryIdSeed;
+ }
+
+ @VisibleForTesting
+ TajoResourceTracker getResourceTracker() {
+ return resourceTracker;
+ }
+
+ public AbstractQueryScheduler getScheduler() {
+ return scheduler;
+ }
+
+ public void submitQuery(QuerySchedulingInfo schedulingInfo) {
+ scheduler.submitQuery(schedulingInfo);
+ }
+
+ public TajoRMContext getRMContext() {
+ return rmContext;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceTracker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceTracker.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceTracker.java
index 2a18de7..3e3e3b4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceTracker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoResourceTracker.java
@@ -20,79 +20,76 @@ package org.apache.tajo.master.rm;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
-import org.apache.tajo.common.exception.NotImplementedException;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.TajoHeartbeatResponse;
import org.apache.tajo.ipc.TajoResourceTrackerProtocol;
+import org.apache.tajo.ipc.TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.scheduler.event.SchedulerEvent;
+import org.apache.tajo.master.scheduler.event.SchedulerEventType;
+import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.rpc.AsyncRpcServer;
import org.apache.tajo.util.NetUtils;
-import org.apache.tajo.util.ProtoUtil;
+import org.apache.tajo.util.TUtil;
-import java.io.IOError;
import java.net.InetSocketAddress;
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.NodeHeartbeat;
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService;
+import static org.apache.tajo.ResourceProtos.*;
/**
- * It receives pings that workers periodically send. The ping messages contains the worker resources and their statuses.
- * From ping messages, {@link TajoResourceTracker} tracks the recent status of all workers.
+ * It receives pings that nodes periodically send. The ping messages contains the node resources and their statuses.
+ * From ping messages, {@link TajoResourceTracker} tracks the recent status of all nodes.
*
* In detail, it has two main roles as follows:
*
* <ul>
* <li>Membership management for nodes which join to a Tajo cluster</li>
* <ul>
- * <li>Register - It receives the ping from a new worker. It registers the worker.</li>
- * <li>Unregister - It unregisters a worker who does not send ping for some expiry time.</li>
+ * <li>Register - It receives the ping from a new node. It registers the node.</li>
+ * <li>Unregister - It unregisters a node who does not send ping for some expiry time.</li>
* <ul>
- * <li>Status Update - It updates the status of all participating workers</li>
+ * <li>Status Update - It updates the status of all participating nodes</li>
* </ul>
*/
public class TajoResourceTracker extends AbstractService implements TajoResourceTrackerProtocolService.Interface {
/** Class logger */
private Log LOG = LogFactory.getLog(TajoResourceTracker.class);
- private final WorkerResourceManager manager;
- /** the context of TajoWorkerResourceManager */
+ private final TajoResourceManager manager;
+ /** the context of TajoResourceManager */
private final TajoRMContext rmContext;
- /** Liveliness monitor which checks ping expiry times of workers */
- private final WorkerLivelinessMonitor workerLivelinessMonitor;
+ /** Liveliness monitor which checks ping expiry times of nodes */
+ private final NodeLivelinessMonitor nodeLivelinessMonitor;
- /** RPC server for worker resource tracker */
+ /** RPC server for node resource tracker */
private AsyncRpcServer server;
- /** The bind address of RPC server of worker resource tracker */
+ /** The bind address of RPC server of node resource tracker */
private InetSocketAddress bindAddress;
- public TajoResourceTracker(WorkerResourceManager manager, WorkerLivelinessMonitor workerLivelinessMonitor) {
+ /** node heartbeat interval in query running */
+ private int activeInterval;
+
+ public TajoResourceTracker(TajoResourceManager manager, NodeLivelinessMonitor nodeLivelinessMonitor) {
super(TajoResourceTracker.class.getSimpleName());
this.manager = manager;
this.rmContext = manager.getRMContext();
- this.workerLivelinessMonitor = workerLivelinessMonitor;
+ this.nodeLivelinessMonitor = nodeLivelinessMonitor;
}
@Override
public void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- TajoConf systemConf = (TajoConf) conf;
+
+ TajoConf systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+ activeInterval = systemConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_ACTIVE_INTERVAL);
String confMasterServiceAddr = systemConf.getVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(confMasterServiceAddr);
- try {
- server = new AsyncRpcServer(TajoResourceTrackerProtocol.class, this, initIsa, 3);
- } catch (Exception e) {
- LOG.error(e);
- throw new IOError(e);
- }
+ int workerNum = systemConf.getIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM);
+ server = new AsyncRpcServer(TajoResourceTrackerProtocol.class, this, initIsa, workerNum);
server.start();
bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
@@ -113,103 +110,97 @@ public class TajoResourceTracker extends AbstractService implements TajoResource
super.serviceStop();
}
- /** The response builder */
- private static final TajoHeartbeatResponse.Builder builder =
- TajoHeartbeatResponse.newBuilder().setHeartbeatResult(ProtoUtil.TRUE);
-
- private static WorkerStatusEvent createStatusEvent(int workerId, NodeHeartbeat heartbeat) {
- return new WorkerStatusEvent(
- workerId,
- heartbeat.getServerStatus().getRunningTaskNum(),
- heartbeat.getServerStatus().getJvmHeap().getMaxHeap(),
- heartbeat.getServerStatus().getJvmHeap().getFreeHeap(),
- heartbeat.getServerStatus().getJvmHeap().getTotalHeap());
+ private static NodeStatusEvent createStatusEvent(NodeHeartbeatRequest heartbeat) {
+ return new NodeStatusEvent(
+ heartbeat.getWorkerId(),
+ heartbeat.getRunningTasks(),
+ heartbeat.getRunningQueryMasters(),
+ new NodeResource(heartbeat.getAvailableResource()),
+ heartbeat.hasTotalResource() ? new NodeResource(heartbeat.getTotalResource()) : null);
}
@Override
- public void heartbeat(
+ public void nodeHeartbeat(
RpcController controller,
- NodeHeartbeat heartbeat,
- RpcCallback<TajoHeartbeatResponse> done) {
+ NodeHeartbeatRequest heartbeat,
+ RpcCallback<NodeHeartbeatResponse> done) {
+ NodeHeartbeatResponse.Builder response = NodeHeartbeatResponse.newBuilder();
+ ResponseCommand responseCommand = ResponseCommand.NORMAL;
try {
// get a workerId from the heartbeat
- int workerId = heartbeat.getConnectionInfo().getId();
+ int workerId = heartbeat.getWorkerId();
- if(rmContext.getWorkers().containsKey(workerId)) { // if worker is running
+ if(rmContext.getNodes().containsKey(workerId)) { // if node is running
// status update
- rmContext.getDispatcher().getEventHandler().handle(createStatusEvent(workerId, heartbeat));
+ rmContext.getDispatcher().getEventHandler().handle(createStatusEvent(heartbeat));
+
+ //refresh scheduler resource
+ rmContext.getDispatcher().getEventHandler().handle(new SchedulerEvent(SchedulerEventType.RESOURCE_UPDATE));
+
// refresh ping
- workerLivelinessMonitor.receivedPing(workerId);
+ nodeLivelinessMonitor.receivedPing(workerId);
- } else if (rmContext.getInactiveWorkers().containsKey(workerId)) { // worker was inactive
+ } else if (rmContext.getInactiveNodes().containsKey(workerId)) { // node was inactive
+ if (!heartbeat.hasConnectionInfo()) {
+ // request membership to worker node
+ responseCommand = ResponseCommand.MEMBERSHIP;
+ } else {
- // remove the inactive worker from the list of inactive workers.
- Worker worker = rmContext.getInactiveWorkers().remove(workerId);
- workerLivelinessMonitor.unregister(worker.getWorkerId());
+ // remove the inactive nodeStatus from the list of inactive nodes.
+ NodeStatus nodeStatus = rmContext.getInactiveNodes().remove(workerId);
+ nodeLivelinessMonitor.unregister(nodeStatus.getWorkerId());
- // create new worker instance
- Worker newWorker = createWorkerResource(heartbeat);
- int newWorkerId = newWorker.getWorkerId();
- // add the new worker to the list of active workers
- rmContext.getWorkers().putIfAbsent(newWorkerId, newWorker);
+ // create new nodeStatus instance
+ NodeStatus newNodeStatus = createNodeStatus(heartbeat);
+ int newWorkerId = newNodeStatus.getWorkerId();
+ // add the new nodeStatus to the list of active nodes
+ rmContext.getNodes().putIfAbsent(newWorkerId, newNodeStatus);
- // Transit the worker to RUNNING
- rmContext.getDispatcher().getEventHandler().handle(new WorkerEvent(newWorkerId, WorkerEventType.STARTED));
- // register the worker to the liveliness monitor
- workerLivelinessMonitor.register(newWorkerId);
+ // Transit the nodeStatus to RUNNING
+ rmContext.getDispatcher().getEventHandler().handle(new NodeEvent(newWorkerId, NodeEventType.STARTED));
+ // register the nodeStatus to the liveliness monitor
+ nodeLivelinessMonitor.register(newWorkerId);
- } else { // if new worker pings firstly
+ rmContext.getDispatcher().getEventHandler().handle(new SchedulerEvent(SchedulerEventType.RESOURCE_UPDATE));
+ }
- // create new worker instance
- Worker newWorker = createWorkerResource(heartbeat);
- Worker oldWorker = rmContext.getWorkers().putIfAbsent(workerId, newWorker);
+ } else { // if new node pings firstly
- if (oldWorker == null) {
- // Transit the worker to RUNNING
- rmContext.rmDispatcher.getEventHandler().handle(new WorkerEvent(workerId, WorkerEventType.STARTED));
+ // The pings have not membership information
+ if (!heartbeat.hasConnectionInfo()) {
+ // request membership to node
+ responseCommand = ResponseCommand.MEMBERSHIP;
} else {
- LOG.info("Reconnect from the node at: " + workerId);
- workerLivelinessMonitor.unregister(workerId);
- rmContext.getDispatcher().getEventHandler().handle(new WorkerReconnectEvent(workerId, newWorker));
- }
- workerLivelinessMonitor.register(workerId);
+ // create new node instance
+ NodeStatus newNodeStatus = createNodeStatus(heartbeat);
+ NodeStatus oldNodeStatus = rmContext.getNodes().putIfAbsent(workerId, newNodeStatus);
+
+ if (oldNodeStatus == null) {
+ // Transit the worker to RUNNING
+ rmContext.rmDispatcher.getEventHandler().handle(new NodeEvent(workerId, NodeEventType.STARTED));
+ } else {
+ LOG.info("Reconnect from the node at: " + workerId);
+ nodeLivelinessMonitor.unregister(workerId);
+ rmContext.getDispatcher().getEventHandler().handle(new NodeReconnectEvent(workerId, newNodeStatus));
+ }
+
+ nodeLivelinessMonitor.register(workerId);
+ rmContext.getDispatcher().getEventHandler().handle(new SchedulerEvent(SchedulerEventType.RESOURCE_UPDATE));
+ }
}
-
} finally {
- builder.setClusterResourceSummary(manager.getClusterResourceSummary());
- done.run(builder.build());
+ if(manager.getScheduler().getRunningQuery() > 0) {
+ response.setHeartBeatInterval(activeInterval);
+ }
+ done.run(response.setCommand(responseCommand).build());
}
}
- @Override
- public void nodeHeartbeat(RpcController controller, TajoResourceTrackerProtocol.NodeHeartbeatRequestProto request,
- RpcCallback<TajoResourceTrackerProtocol.NodeHeartbeatResponseProto> done) {
- //TODO implement with ResourceManager for scheduler
- TajoResourceTrackerProtocol.NodeHeartbeatResponseProto.Builder
- response = TajoResourceTrackerProtocol.NodeHeartbeatResponseProto.newBuilder();
- done.run(response.setCommand(TajoResourceTrackerProtocol.ResponseCommand.NORMAL).build());
- }
-
- private Worker createWorkerResource(NodeHeartbeat request) {
- WorkerResource workerResource = new WorkerResource();
-
- if(request.getServerStatus() != null) {
- workerResource.setMemoryMB(request.getServerStatus().getMemoryResourceMB());
- workerResource.setCpuCoreSlots(request.getServerStatus().getSystem().getAvailableProcessors());
- workerResource.setDiskSlots(request.getServerStatus().getDiskSlots());
- workerResource.setNumRunningTasks(request.getServerStatus().getRunningTaskNum());
- workerResource.setMaxHeap(request.getServerStatus().getJvmHeap().getMaxHeap());
- workerResource.setFreeHeap(request.getServerStatus().getJvmHeap().getFreeHeap());
- workerResource.setTotalHeap(request.getServerStatus().getJvmHeap().getTotalHeap());
- } else {
- workerResource.setMemoryMB(4096);
- workerResource.setDiskSlots(4);
- workerResource.setCpuCoreSlots(4);
- }
-
- return new Worker(rmContext, workerResource, new WorkerConnectionInfo(request.getConnectionInfo()));
+ private NodeStatus createNodeStatus(NodeHeartbeatRequest request) {
+ return new NodeStatus(rmContext, new NodeResource(request.getTotalResource()),
+ new WorkerConnectionInfo(request.getConnectionInfo()));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
deleted file mode 100644
index 8c5b96c..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import org.apache.hadoop.yarn.api.records.*;
-import org.apache.tajo.master.container.TajoContainer;
-import org.apache.tajo.master.container.TajoContainerId;
-
-
-public class TajoWorkerContainer extends TajoContainer {
- TajoContainerId id;
- NodeId nodeId;
- Worker worker;
-
- public Worker getWorkerResource() {
- return worker;
- }
-
- public void setWorkerResource(Worker workerResource) {
- this.worker = workerResource;
- }
-
- @Override
- public TajoContainerId getId() {
- return id;
- }
-
- @Override
- public void setId(TajoContainerId id) {
- this.id = id;
- }
-
- @Override
- public NodeId getNodeId() {
- return nodeId;
- }
-
- @Override
- public void setNodeId(NodeId nodeId) {
- this.nodeId = nodeId;
- }
-
- @Override
- public String getNodeHttpAddress() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setNodeHttpAddress(String nodeHttpAddress) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Resource getResource() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setResource(Resource resource) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Priority getPriority() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setPriority(Priority priority) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Token getContainerToken() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setContainerToken(Token containerToken) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public int compareTo(TajoContainer container) {
- return getId().compareTo(container.getId());
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TajoWorkerContainer that = (TajoWorkerContainer) o;
-
- if (id != null ? !id.equals(that.id) : that.id != null) return false;
- if (nodeId != null ? !nodeId.equals(that.nodeId) : that.nodeId != null) return false;
- if (worker != null ? !worker.equals(that.worker) : that.worker != null) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = id != null ? id.hashCode() : 0;
- result = 31 * result + (nodeId != null ? nodeId.hashCode() : 0);
- result = 31 * result + (worker != null ? worker.hashCode() : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
deleted file mode 100644
index 184de71..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.rm;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.proto.YarnProtos;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.master.container.TajoContainerId;
-
-public class TajoWorkerContainerId extends TajoContainerId {
- ApplicationAttemptId applicationAttemptId;
- int id;
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return applicationAttemptId;
- }
-
- @Override
- public void setApplicationAttemptId(ApplicationAttemptId atId) {
- this.applicationAttemptId = atId;
- }
-
- @Override
- public int getId() {
- return id;
- }
-
- @Override
- public void setId(int id) {
- this.id = id;
- }
-
- public ContainerProtocol.TajoContainerIdProto getProto() {
- YarnProtos.ApplicationIdProto appIdProto = YarnProtos.ApplicationIdProto.newBuilder()
- .setClusterTimestamp(applicationAttemptId.getApplicationId().getClusterTimestamp())
- .setId(applicationAttemptId.getApplicationId().getId())
- .build();
-
- YarnProtos.ApplicationAttemptIdProto attemptIdProto = YarnProtos.ApplicationAttemptIdProto.newBuilder()
- .setAttemptId(applicationAttemptId.getAttemptId())
- .setApplicationId(appIdProto)
- .build();
-
- return ContainerProtocol.TajoContainerIdProto.newBuilder()
- .setAppAttemptId(attemptIdProto)
- .setAppId(appIdProto)
- .setId(id)
- .build();
- }
-
- public static ContainerProtocol.TajoContainerIdProto getContainerIdProto(TajoContainerId containerId) {
- if(containerId instanceof TajoWorkerContainerId) {
- return ((TajoWorkerContainerId)containerId).getProto();
- } else {
- YarnProtos.ApplicationIdProto appIdProto = YarnProtos.ApplicationIdProto.newBuilder()
- .setClusterTimestamp(containerId.getApplicationAttemptId().getApplicationId().getClusterTimestamp())
- .setId(containerId.getApplicationAttemptId().getApplicationId().getId())
- .build();
-
- YarnProtos.ApplicationAttemptIdProto attemptIdProto = YarnProtos.ApplicationAttemptIdProto.newBuilder()
- .setAttemptId(containerId.getApplicationAttemptId().getAttemptId())
- .setApplicationId(appIdProto)
- .build();
-
- return ContainerProtocol.TajoContainerIdProto.newBuilder()
- .setAppAttemptId(attemptIdProto)
- .setAppId(appIdProto)
- .setId(containerId.getId())
- .build();
- }
- }
-
- @Override
- protected void build() {
-
- }
-}
[04/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java
index 299952e..5e1ccc1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java
@@ -25,44 +25,44 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TajoProtos;
import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.TaskId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.engine.query.TaskRequest;
import org.apache.tajo.engine.query.TaskRequestImpl;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.resource.NodeResource;
-import org.apache.tajo.resource.NodeResources;
-import org.apache.tajo.worker.event.*;
+import org.apache.tajo.util.TUtil;
+import org.apache.tajo.worker.event.NodeResourceDeallocateEvent;
+import org.apache.tajo.worker.event.NodeResourceEvent;
+import org.apache.tajo.worker.event.TaskStartEvent;
+import org.apache.tajo.ResourceProtos.TaskRequestProto;
import java.io.IOException;
import java.util.Map;
-import java.util.concurrent.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
/**
* TaskExecutor uses a number of threads equal to the number of slots available for running tasks on the Worker
*/
-public class TaskExecutor extends AbstractService implements EventHandler<TaskExecutorEvent> {
+public class TaskExecutor extends AbstractService implements EventHandler<TaskStartEvent> {
private static final Log LOG = LogFactory.getLog(TaskExecutor.class);
- private final TaskManager taskManager;
- private final EventHandler rmEventHandler;
+ private final TajoWorker.WorkerContext workerContext;
private final Map<TaskAttemptId, NodeResource> allocatedResourceMap;
private final BlockingQueue<Task> taskQueue;
private final AtomicInteger runningTasks;
- private ThreadPoolExecutor fetcherExecutor;
+ private ExecutorService fetcherThreadPool;
private ExecutorService threadPool;
private TajoConf tajoConf;
private volatile boolean isStopped;
- public TaskExecutor(TaskManager taskManager, EventHandler rmEventHandler) {
+ public TaskExecutor(TajoWorker.WorkerContext workerContext) {
super(TaskExecutor.class.getName());
- this.taskManager = taskManager;
- this.rmEventHandler = rmEventHandler;
+ this.workerContext = workerContext;
this.allocatedResourceMap = Maps.newConcurrentMap();
this.runningTasks = new AtomicInteger();
this.taskQueue = new LinkedBlockingQueue<Task>();
@@ -70,12 +70,8 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskEx
@Override
protected void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- this.tajoConf = (TajoConf) conf;
- this.taskManager.getDispatcher().register(TaskExecutorEvent.EventType.class, this);
+ this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
super.serviceInit(conf);
}
@@ -85,12 +81,9 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskEx
this.threadPool = Executors.newFixedThreadPool(nThreads,
new ThreadFactoryBuilder().setNameFormat("Task executor #%d").build());
- //TODO move to tajoConf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM);
- int maxFetcherThreads = Runtime.getRuntime().availableProcessors() * 2;
- this.fetcherExecutor = new ThreadPoolExecutor(Math.min(nThreads, maxFetcherThreads),
- maxFetcherThreads,
- 60L, TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(true));
+ int maxFetcherThreads = tajoConf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM);
+ this.fetcherThreadPool = Executors.newFixedThreadPool(nThreads,
+ new ThreadFactoryBuilder().setNameFormat("Fetcher executor #%d").build());
for (int i = 0; i < nThreads; i++) {
@@ -106,7 +99,7 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskEx
isStopped = true;
threadPool.shutdown();
- fetcherExecutor.shutdown();
+ fetcherThreadPool.shutdown();
super.serviceStop();
}
@@ -131,19 +124,28 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskEx
return task;
}
- @SuppressWarnings("unchecked")
protected void stopTask(TaskAttemptId taskId) {
runningTasks.decrementAndGet();
- rmEventHandler.handle(new NodeResourceDeallocateEvent(allocatedResourceMap.remove(taskId)));
+ releaseResource(taskId);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void releaseResource(TaskAttemptId taskId) {
+ NodeResource resource = allocatedResourceMap.remove(taskId);
+
+ if(resource != null) {
+ workerContext.getNodeResourceManager().getDispatcher().getEventHandler().handle(
+ new NodeResourceDeallocateEvent(resource, NodeResourceEvent.ResourceType.TASK));
+ }
}
protected ExecutorService getFetcherExecutor() {
- return fetcherExecutor;
+ return fetcherThreadPool;
}
protected Task createTask(ExecutionBlockContext executionBlockContext,
- TajoWorkerProtocol.TaskRequestProto taskRequest) throws IOException {
+ TaskRequestProto taskRequest) throws IOException {
Task task = null;
TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
if (executionBlockContext.getTasks().containsKey(taskAttemptId)) {
@@ -158,37 +160,34 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskEx
}
@Override
- public void handle(TaskExecutorEvent event) {
-
- if (event instanceof TaskStartEvent) {
- TaskStartEvent startEvent = (TaskStartEvent) event;
- allocatedResourceMap.put(startEvent.getTaskId(), startEvent.getAllocatedResource());
-
- ExecutionBlockContext context = taskManager.getExecutionBlockContext(
- startEvent.getTaskId().getTaskId().getExecutionBlockId());
-
- try {
- Task task = createTask(context, startEvent.getTaskRequest());
- if (task != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Arrival task: " + task.getTaskContext().getTaskId() +
- ", allocated resource: " + startEvent.getAllocatedResource());
- }
- taskQueue.put(task);
- runningTasks.incrementAndGet();
- context.getWorkerContext().getWorkerSystemMetrics()
- .histogram("tasks", "running").update(runningTasks.get());
- } else {
- LOG.warn("Release duplicate task resource: " + startEvent.getAllocatedResource());
- stopTask(startEvent.getTaskId());
- }
- } catch (InterruptedException e) {
- if (!isStopped) {
- LOG.fatal(e.getMessage(), e);
+ public void handle(TaskStartEvent event) {
+
+ allocatedResourceMap.put(event.getTaskAttemptId(), event.getAllocatedResource());
+
+ ExecutionBlockContext context = workerContext.getTaskManager().getExecutionBlockContext(
+ event.getTaskAttemptId().getTaskId().getExecutionBlockId());
+
+ try {
+ Task task = createTask(context, event.getTaskRequest());
+ if (task != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Arrival task: " + task.getTaskContext().getTaskId() +
+ ", allocated resource: " + event.getAllocatedResource());
}
- } catch (IOException e) {
- stopTask(startEvent.getTaskId());
+ taskQueue.put(task);
+ runningTasks.incrementAndGet();
+ context.getWorkerContext().getWorkerSystemMetrics()
+ .histogram("tasks", "running").update(runningTasks.get());
+ } else {
+ LOG.warn("Release duplicate task resource: " + event.getAllocatedResource());
+ stopTask(event.getTaskAttemptId());
+ }
+ } catch (InterruptedException e) {
+ if (!isStopped) {
+ LOG.fatal(e.getMessage(), e);
}
+ } catch (Exception e) {
+ stopTask(event.getTaskAttemptId());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
index c2432eb..52b0d0b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
@@ -29,8 +29,8 @@ import java.util.Collections;
import java.util.List;
import static org.apache.tajo.TajoProtos.TaskAttemptState;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.FetcherHistoryProto;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskHistoryProto;
+import static org.apache.tajo.ResourceProtos.FetcherHistoryProto;
+import static org.apache.tajo.ResourceProtos.TaskHistoryProto;
/**
* The history class for Task processing.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java
index 7697458..d77c583 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java
@@ -37,15 +37,17 @@ import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.physical.PhysicalExec;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.engine.query.TaskRequest;
import org.apache.tajo.ipc.QueryMasterProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.*;
-import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.plan.serder.PlanProto.ShuffleType;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty.EnforceType;
import org.apache.tajo.plan.function.python.TajoScriptEngine;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
@@ -65,8 +67,7 @@ import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
-import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
+import static org.apache.tajo.ResourceProtos.*;
public class TaskImpl implements Task {
private static final Log LOG = LogFactory.getLog(TaskImpl.class);
@@ -466,6 +467,7 @@ public class TaskImpl implements Task {
@Override
public void cleanup() {
+ // history store in memory while running stage
TaskHistory taskHistory = createTaskHistory();
executionBlockContext.addTaskHistory(getId().getTaskId(), taskHistory);
executionBlockContext.getTasks().remove(getId());
@@ -485,6 +487,7 @@ public class TaskImpl implements Task {
stopScriptExecutors();
}
+ @Override
public TaskHistory createTaskHistory() {
TaskHistory taskHistory = null;
try {
@@ -508,16 +511,12 @@ public class TaskImpl implements Task {
int i = 0;
FetcherHistoryProto.Builder builder = FetcherHistoryProto.newBuilder();
for (Fetcher fetcher : fetcherRunners) {
- // TODO store the fetcher histories
- if (systemConf.getBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED)) {
- builder.setStartTime(fetcher.getStartTime());
- builder.setFinishTime(fetcher.getFinishTime());
- builder.setFileLength(fetcher.getFileLen());
- builder.setMessageReceivedCount(fetcher.getMessageReceiveCount());
- builder.setState(fetcher.getState());
-
- taskHistory.addFetcherHistory(builder.build());
- }
+ builder.setStartTime(fetcher.getStartTime());
+ builder.setFinishTime(fetcher.getFinishTime());
+ builder.setFileLength(fetcher.getFileLen());
+ builder.setMessageReceivedCount(fetcher.getMessageReceiveCount());
+ builder.setState(fetcher.getState());
+ taskHistory.addFetcherHistory(builder.build());
if (fetcher.getState() == TajoProtos.FetcherState.FETCH_FINISHED) i++;
}
taskHistory.setFinishedFetchCount(i);
@@ -529,6 +528,10 @@ public class TaskImpl implements Task {
return taskHistory;
}
+ public List<Fetcher> getFetchers() {
+ return fetcherRunners;
+ }
+
public int hashCode() {
return context.hashCode();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java
index 7990a72..f518fd3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java
@@ -29,12 +29,24 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TaskId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ipc.QueryMasterProtocol;
+import org.apache.tajo.rpc.AsyncRpcClient;
+import org.apache.tajo.rpc.CallFuture;
+import org.apache.tajo.rpc.RpcClientManager;
+import org.apache.tajo.rpc.RpcConstants;
+import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.event.*;
import java.io.IOException;
-import java.util.*;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.tajo.ResourceProtos.ExecutionBlockListProto;
+import static org.apache.tajo.ResourceProtos.ExecutionBlockContextRequest;
+import static org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse;
/**
* A TaskManager is responsible for managing executionBlock resource and tasks.
@@ -45,26 +57,23 @@ public class TaskManager extends AbstractService implements EventHandler<TaskMan
private final TajoWorker.WorkerContext workerContext;
private final Map<ExecutionBlockId, ExecutionBlockContext> executionBlockContextMap;
private final Dispatcher dispatcher;
- private final EventHandler rmEventHandler;
+ private TaskExecutor executor;
- private TajoConf tajoConf;
+ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext){
+ this(dispatcher, workerContext, null);
+ }
- public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext, EventHandler rmEventHandler) {
+ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext, TaskExecutor executor) {
super(TaskManager.class.getName());
this.dispatcher = dispatcher;
this.workerContext = workerContext;
this.executionBlockContextMap = Maps.newHashMap();
- this.rmEventHandler = rmEventHandler;
+ this.executor = executor;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
-
- this.tajoConf = (TajoConf)conf;
dispatcher.register(TaskManagerEvent.EventType.class, this);
super.serviceInit(conf);
}
@@ -87,22 +96,51 @@ public class TaskManager extends AbstractService implements EventHandler<TaskMan
return workerContext;
}
- protected ExecutionBlockContext createExecutionBlock(TajoWorkerProtocol.RunExecutionBlockRequestProto request) {
+ protected TaskExecutor getTaskExecutor() {
+ if (executor == null) {
+ executor = workerContext.getTaskExecuor();
+ }
+ return executor;
+ }
+
+ public int getRunningTasks() {
+ return workerContext.getTaskExecuor().getRunningTasks();
+ }
+
+ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionBlockId,
+ String queryMasterHostAndPort) {
+
+ LOG.info("QueryMaster Address:" + queryMasterHostAndPort);
+
+ AsyncRpcClient client = null;
try {
- ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), null, request);
+ InetSocketAddress address = NetUtils.createSocketAddr(queryMasterHostAndPort);
+ ExecutionBlockContextRequest.Builder request = ExecutionBlockContextRequest.newBuilder();
+ request.setExecutionBlockId(executionBlockId.getProto())
+ .setWorker(getWorkerContext().getConnectionInfo().getProto());
+
+ client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true);
+ QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub();
+ CallFuture<ExecutionBlockContextResponse> callback = new CallFuture<ExecutionBlockContextResponse>();
+ stub.getExecutionBlockContext(callback.getController(), request.build(), callback);
+
+ ExecutionBlockContextResponse contextProto =
+ callback.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), contextProto, client);
context.init();
return context;
} catch (Throwable e) {
+ RpcClientManager.cleanup(client);
LOG.fatal(e.getMessage(), e);
throw new RuntimeException(e);
}
}
protected void stopExecutionBlock(ExecutionBlockContext context,
- TajoWorkerProtocol.ExecutionBlockListProto cleanupList) {
+ ExecutionBlockListProto cleanupList) {
- if(context != null){
+ if (context != null) {
try {
context.getSharedResource().releaseBroadcastCache(context.getExecutionBlockId());
context.sendShuffleReport();
@@ -127,22 +165,72 @@ public class TaskManager extends AbstractService implements EventHandler<TaskMan
@Override
public void handle(TaskManagerEvent event) {
- LOG.info("======================== Processing " + event.getExecutionBlockId() + " of type " + event.getType());
- if (event instanceof ExecutionBlockStartEvent) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("======================== Processing " + event + " of type " + event.getType());
+ }
+
+ switch (event.getType()) {
+ case TASK_START: {
+ //receive event from NodeResourceManager
+ TaskStartEvent taskStartEvent = TUtil.checkTypeAndGet(event, TaskStartEvent.class);
+ try {
+ if (!executionBlockContextMap.containsKey(taskStartEvent.getExecutionBlockId())) {
+ ExecutionBlockContext context = createExecutionBlock(taskStartEvent.getExecutionBlockId(),
+ taskStartEvent.getTaskRequest().getQueryMasterHostAndPort());
- //receive event from NodeResourceManager
- if(!executionBlockContextMap.containsKey(event.getExecutionBlockId())) {
- ExecutionBlockContext context = createExecutionBlock(((ExecutionBlockStartEvent) event).getRequestProto());
- executionBlockContextMap.put(context.getExecutionBlockId(), context);
- } else {
- LOG.warn("Already initialized ExecutionBlock: " + event.getExecutionBlockId());
+ executionBlockContextMap.put(context.getExecutionBlockId(), context);
+ LOG.info("Running ExecutionBlocks: " + executionBlockContextMap.size()
+ + ", running tasks:" + getRunningTasks() + ", availableResource: "
+ + workerContext.getNodeResourceManager().getAvailableResource());
+ }
+ getTaskExecutor().handle(taskStartEvent);
+ } catch (Exception e) {
+ getTaskExecutor().releaseResource(taskStartEvent.getTaskAttemptId());
+ getWorkerContext().getTaskManager().getDispatcher().getEventHandler()
+ .handle(new ExecutionBlockErrorEvent(taskStartEvent.getExecutionBlockId(), e));
+ }
+ break;
+ }
+ case EB_STOP: {
+ //receive event from QueryMaster
+ ExecutionBlockStopEvent executionBlockStopEvent = TUtil.checkTypeAndGet(event, ExecutionBlockStopEvent.class);
+ workerContext.getNodeResourceManager().getDispatcher().getEventHandler()
+ .handle(new NodeStatusEvent(NodeStatusEvent.EventType.FLUSH_REPORTS));
+ stopExecutionBlock(executionBlockContextMap.remove(executionBlockStopEvent.getExecutionBlockId()),
+ executionBlockStopEvent.getCleanupList());
+ break;
+ }
+ case QUERY_STOP: {
+ QueryStopEvent queryStopEvent = TUtil.checkTypeAndGet(event, QueryStopEvent.class);
+
+ //cleanup failure ExecutionBlock
+ for (ExecutionBlockId ebId : executionBlockContextMap.keySet()) {
+ if (ebId.getQueryId().equals(queryStopEvent.getQueryId())) {
+ try {
+ executionBlockContextMap.remove(ebId).stop();
+ } catch (Exception e) {
+ LOG.fatal(e.getMessage(), e);
+ }
+ }
+ }
+ workerContext.cleanup(queryStopEvent.getQueryId().toString());
+ break;
}
- } else if (event instanceof ExecutionBlockStopEvent) {
- //receive event from QueryMaster
- rmEventHandler.handle(new NodeStatusEvent(NodeStatusEvent.EventType.FLUSH_REPORTS));
- stopExecutionBlock(executionBlockContextMap.remove(event.getExecutionBlockId()),
- ((ExecutionBlockStopEvent) event).getCleanupList());
+ case EB_FAIL: {
+ ExecutionBlockErrorEvent errorEvent = TUtil.checkTypeAndGet(event, ExecutionBlockErrorEvent.class);
+ LOG.error(errorEvent.getError().getMessage(), errorEvent.getError());
+ ExecutionBlockContext context = executionBlockContextMap.remove(errorEvent.getExecutionBlockId());
+
+ if (context != null) {
+ context.getSharedResource().releaseBroadcastCache(context.getExecutionBlockId());
+ getWorkerContext().getTaskHistoryWriter().flushTaskHistories();
+ context.stop();
+ }
+ break;
+ }
+ default:
+ break;
}
}
@@ -158,17 +246,14 @@ public class TaskManager extends AbstractService implements EventHandler<TaskMan
return null;
}
- public List<TaskHistory> getTaskHistories(ExecutionBlockId executionblockId) throws IOException {
- List<TaskHistory> histories = new ArrayList<TaskHistory>();
- ExecutionBlockContext context = executionBlockContextMap.get(executionblockId);
- if (context != null) {
- histories.addAll(context.getTaskHistories().values());
- }
- //TODO get List<TaskHistory> from HistoryReader
- return histories;
+ public List<org.apache.tajo.util.history.TaskHistory> getTaskHistories(ExecutionBlockId executionblockId)
+ throws IOException {
+
+ return getWorkerContext().getHistoryReader().getTaskHistory(executionblockId.getQueryId().toString(),
+ executionblockId.toString());
}
- public TaskHistory getTaskHistory(TaskId taskId) {
+ public TaskHistory getTaskHistory(TaskId taskId) throws IOException {
TaskHistory history = null;
ExecutionBlockContext context = executionBlockContextMap.get(taskId.getExecutionBlockId());
if (context != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
deleted file mode 100644
index 207b47e..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.engine.query.TaskRequestImpl;
-import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService;
-import org.apache.tajo.master.container.TajoContainerId;
-import org.apache.tajo.master.container.TajoContainerIdPBImpl;
-import org.apache.tajo.master.container.TajoConverterUtils;
-import org.apache.tajo.rpc.CallFuture;
-import org.apache.tajo.rpc.NullCallback;
-
-import java.util.concurrent.*;
-
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
-
-/**
- * The driver class for Tajo Task processing.
- */
-@Deprecated
-public class TaskRunner extends AbstractService {
- /** class logger */
- private static final Log LOG = LogFactory.getLog(TaskRunner.class);
-
- private TajoConf systemConf;
-
- private volatile boolean stopped = false;
- private Path baseDirPath;
-
- private TajoContainerId containerId;
-
- // for Fetcher
- private ExecutorService fetchLauncher;
-
- // A thread to receive each assigned query unit and execute the query unit
- private Thread taskLauncher;
-
- // Contains the object references related for TaskRunner
- private ExecutionBlockContext executionBlockContext;
-
- private long finishTime;
-
- private TaskRunnerHistory history;
-
- public TaskRunner(ExecutionBlockContext executionBlockContext, String containerId) {
- super(TaskRunner.class.getName());
-
- ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
- ThreadFactory fetcherFactory = builder.setNameFormat("Fetcher executor #%d").build();
- this.systemConf = executionBlockContext.getConf();
- this.fetchLauncher = Executors.newFixedThreadPool(
- systemConf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM), fetcherFactory);
- try {
- this.containerId = TajoConverterUtils.toTajoContainerId(containerId);
- this.executionBlockContext = executionBlockContext;
- this.history = executionBlockContext.createTaskRunnerHistory(this);
- this.history.setState(getServiceState());
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- // TODO this is expensive. we should change to unique id
- public String getId() {
- return getId(getContext().getExecutionBlockId(), containerId);
- }
-
- public TajoContainerId getContainerId(){
- return containerId;
- }
-
- public static String getId(ExecutionBlockId executionBlockId, TajoContainerId containerId) {
- return executionBlockId + "," + containerId;
- }
-
- public TaskRunnerHistory getHistory(){
- return history;
- }
-
- public Path getTaskBaseDir(){
- return baseDirPath;
- }
-
- public ExecutorService getFetchLauncher() {
- return fetchLauncher;
- }
-
- @Override
- public void init(Configuration conf) {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("conf should be a TajoConf Type.");
- }
- this.systemConf = (TajoConf)conf;
-
- try {
- // the base dir for an output dir
- baseDirPath = getContext().createBaseDir();
- LOG.info("TaskRunner basedir is created (" + baseDirPath +")");
- } catch (Throwable t) {
- t.printStackTrace();
- LOG.error(t, t);
- }
- super.init(conf);
- this.history.setState(getServiceState());
- }
-
- @Override
- public void start() {
- super.start();
- history.setStartTime(getStartTime());
- this.history.setState(getServiceState());
- run();
- }
-
- @Override
- public void stop() {
- if(isStopped()) {
- return;
- }
- this.finishTime = System.currentTimeMillis();
- this.history.setFinishTime(finishTime);
- // If this flag become true, taskLauncher will be terminated.
-
- LOG.info("Stop TaskRunner: " + getId());
- synchronized (this) {
- this.stopped = true;
-
- fetchLauncher.shutdown();
- fetchLauncher = null;
-
- notifyAll();
- }
-
- super.stop();
- this.history.setState(getServiceState());
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- public ExecutionBlockContext getContext() {
- return executionBlockContext;
- }
-
- static void fatalError(QueryMasterProtocolService.Interface qmClientService,
- TaskAttemptId taskAttemptId, String message) {
- if (message == null) {
- message = "No error message";
- }
- TaskFatalErrorReport.Builder builder = TaskFatalErrorReport.newBuilder()
- .setId(taskAttemptId.getProto())
- .setErrorMessage(message);
-
- qmClientService.fatalError(null, builder.build(), NullCallback.get());
- }
-
- public void run() {
- LOG.info("TaskRunner startup");
- try {
-
- taskLauncher = new Thread(new Runnable() {
-
- @Override
- public void run() {
- int receivedNum = 0;
- CallFuture<TaskRequestProto> callFuture = null;
- TaskRequestProto taskRequest = null;
-
- while(!stopped && !executionBlockContext.isStopped()) {
- QueryMasterProtocolService.Interface qmClientService = executionBlockContext.getStub();
-
- try {
- if (callFuture == null) {
- callFuture = new CallFuture<TaskRequestProto>();
- LOG.info("Request GetTask: " + getId());
- GetTaskRequestProto request = GetTaskRequestProto.newBuilder()
- .setExecutionBlockId(getExecutionBlockId().getProto())
- .setContainerId(((TajoContainerIdPBImpl) containerId).getProto())
- .setWorkerId(getContext().getWorkerContext().getConnectionInfo().getId())
- .build();
-
- qmClientService.getTask(callFuture.getController(), request, callFuture);
- }
- try {
- // wait for an assigning task for 3 seconds
- taskRequest = callFuture.get(3, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- if(stopped) {
- break;
- }
- } catch (TimeoutException te) {
- if(stopped) {
- break;
- }
- // if there has been no assigning task for a given period,
- // TaskRunner will retry to request an assigning task.
- if (LOG.isDebugEnabled()) {
- LOG.info("Retry assigning task:" + getId());
- }
- continue;
- } catch (ExecutionException ee) {
- if(!getContext().isStopped()){
- LOG.error(ee.getMessage(), ee);
- } else {
- /* EB is stopped */
- break;
- }
- }
-
- if (taskRequest != null) {
- // QueryMaster can send the terminal signal to TaskRunner.
- // If TaskRunner receives the terminal signal, TaskRunner will be terminated
- // immediately.
- if (taskRequest.getShouldDie()) {
- LOG.info("Received ShouldDie flag:" + getId());
- break;
- } else {
- getContext().getWorkerContext().getWorkerSystemMetrics().counter("query", "task").inc();
- LOG.info("Accumulated Received Task: " + (++receivedNum));
-
- TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
- if (getContext().getTasks().containsKey(taskAttemptId)) {
- LOG.error("Duplicate Task Attempt: " + taskAttemptId);
- fatalError(qmClientService, taskAttemptId, "Duplicate Task Attempt: " + taskAttemptId);
- continue;
- }
-
- LOG.info("Initializing: " + taskAttemptId);
- Task task = null;
- try {
- task = new LegacyTaskImpl(getId(), getTaskBaseDir(), taskAttemptId, executionBlockContext,
- new TaskRequestImpl(taskRequest));
- getContext().getTasks().put(taskAttemptId, task);
-
- task.init();
- if (task.hasFetchPhase()) {
- task.fetch(); // The fetch is performed in an asynchronous way.
- }
- // task.run() is a blocking call.
- task.run();
- } catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- fatalError(qmClientService, taskAttemptId, t.getMessage());
- } finally {
- if(task != null) {
- task.cleanup();
- }
-
- callFuture = null;
- taskRequest = null;
- }
- }
- }
- } catch (Throwable t) {
- LOG.fatal(t.getMessage(), t);
- }
- }
- stop();
- //notify to TaskRunnerManager
- getContext().stopTaskRunner(getId());
- }
- });
- taskLauncher.start();
- } catch (Throwable t) {
- LOG.fatal("Unhandled exception. Starting shutdown.", t);
- }
- }
-
- /**
- * @return true if a stop has been requested.
- */
- public boolean isStopped() {
- return this.stopped;
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return getContext().getExecutionBlockId();
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
deleted file mode 100644
index 16d32d4..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.service.Service;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.master.container.TajoContainerId;
-import org.apache.tajo.master.container.TajoConverterUtils;
-
-import java.util.Collections;
-import java.util.Map;
-
-import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskHistoryProto;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskRunnerHistoryProto;
-
-/**
- * The history class for TaskRunner processing.
- */
-@Deprecated
-public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
-
- private Service.STATE state;
- private TajoContainerId containerId;
- private long startTime;
- private long finishTime;
- private ExecutionBlockId executionBlockId;
- private Map<TaskAttemptId, TaskHistory> taskHistoryMap = null;
-
- public TaskRunnerHistory(TajoContainerId containerId, ExecutionBlockId executionBlockId) {
- init();
- this.containerId = containerId;
- this.executionBlockId = executionBlockId;
- }
-
- public TaskRunnerHistory(TaskRunnerHistoryProto proto) {
- this.state = Service.STATE.valueOf(proto.getState());
- this.containerId = TajoConverterUtils.toTajoContainerId(proto.getContainerId());
- this.startTime = proto.getStartTime();
- this.finishTime = proto.getFinishTime();
- this.executionBlockId = new ExecutionBlockId(proto.getExecutionBlockId());
- this.taskHistoryMap = Maps.newTreeMap();
- for (TaskHistoryProto taskHistoryProto : proto.getTaskHistoriesList()) {
- TaskHistory taskHistory = new TaskHistory(taskHistoryProto);
- taskHistoryMap.put(taskHistory.getTaskAttemptId(), taskHistory);
- }
- }
-
- private void init() {
- this.taskHistoryMap = Maps.newHashMap();
- }
-
- public int size() {
- return this.taskHistoryMap.size();
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(containerId, executionBlockId, state, startTime,
- finishTime, taskHistoryMap.size());
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof TaskRunnerHistory) {
- TaskRunnerHistory other = (TaskRunnerHistory) o;
- return getProto().equals(other.getProto());
- }
- return false;
- }
-
- @Override
- public TaskRunnerHistoryProto getProto() {
- TaskRunnerHistoryProto.Builder builder = TaskRunnerHistoryProto.newBuilder();
- builder.setContainerId(containerId.toString());
- builder.setState(state.toString());
- builder.setExecutionBlockId(executionBlockId.getProto());
- builder.setStartTime(startTime);
- builder.setFinishTime(finishTime);
- for (TaskHistory taskHistory : taskHistoryMap.values()){
- builder.addTaskHistories(taskHistory.getProto());
- }
- return builder.build();
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
- }
-
- public Service.STATE getState() {
- return state;
- }
-
- public void setState(Service.STATE state) {
- this.state = state;
- }
-
- public TajoContainerId getContainerId() {
- return containerId;
- }
-
- public void setContainerId(TajoContainerId containerId) {
- this.containerId = containerId;
- }
-
- public TaskHistory getTaskHistory(TaskAttemptId taskAttemptId) {
- return taskHistoryMap.get(taskAttemptId);
- }
-
- public Map<TaskAttemptId, TaskHistory> getTaskHistoryMap() {
- return Collections.unmodifiableMap(taskHistoryMap);
- }
-
- public void addTaskHistory(TaskAttemptId taskAttemptId, TaskHistory taskHistory) {
- taskHistoryMap.put(taskAttemptId, taskHistory);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
deleted file mode 100644
index d18a262..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TaskAttemptId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.worker.event.TaskRunnerEvent;
-import org.apache.tajo.worker.event.TaskRunnerStartEvent;
-import org.apache.tajo.worker.event.TaskRunnerStopEvent;
-
-import java.util.*;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-@Deprecated
-public class TaskRunnerManager extends CompositeService implements EventHandler<TaskRunnerEvent> {
- private static final Log LOG = LogFactory.getLog(TaskRunnerManager.class);
-
- private final ConcurrentMap<ExecutionBlockId, ExecutionBlockContext> executionBlockContextMap = Maps.newConcurrentMap();
- private final ConcurrentMap<String, TaskRunner> taskRunnerMap = Maps.newConcurrentMap();
- private final ConcurrentMap<String, TaskRunnerHistory> taskRunnerHistoryMap = Maps.newConcurrentMap();
- private TajoWorker.WorkerContext workerContext;
- private TajoConf tajoConf;
- private AtomicBoolean stop = new AtomicBoolean(false);
- private FinishedTaskCleanThread finishedTaskCleanThread;
- private Dispatcher dispatcher;
-
- public TaskRunnerManager(TajoWorker.WorkerContext workerContext, Dispatcher dispatcher) {
- super(TaskRunnerManager.class.getName());
-
- this.workerContext = workerContext;
- this.dispatcher = dispatcher;
- }
-
- public TajoWorker.WorkerContext getWorkerContext() {
- return workerContext;
- }
-
- @Override
- public void init(Configuration conf) {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- tajoConf = (TajoConf)conf;
- dispatcher.register(TaskRunnerEvent.EventType.class, this);
- super.init(tajoConf);
- }
-
- @Override
- public void start() {
- finishedTaskCleanThread = new FinishedTaskCleanThread();
- finishedTaskCleanThread.start();
- super.start();
- }
-
- @Override
- public void stop() {
- if(stop.getAndSet(true)) {
- return;
- }
-
- synchronized(taskRunnerMap) {
- for(TaskRunner eachTaskRunner: taskRunnerMap.values()) {
- if(!eachTaskRunner.isStopped()) {
- eachTaskRunner.stop();
- }
- }
- }
- for(ExecutionBlockContext context: executionBlockContextMap.values()) {
- context.stop();
- }
-
- if(finishedTaskCleanThread != null) {
- finishedTaskCleanThread.interrupt();
- }
-
- super.stop();
- }
-
- public void stopTaskRunner(String id) {
- LOG.info("Stop Task:" + id);
- TaskRunner taskRunner = taskRunnerMap.remove(id);
- taskRunner.stop();
- }
-
- public Collection<TaskRunner> getTaskRunners() {
- return Collections.unmodifiableCollection(taskRunnerMap.values());
- }
-
- public Collection<TaskRunnerHistory> getExecutionBlockHistories() {
- return Collections.unmodifiableCollection(taskRunnerHistoryMap.values());
- }
-
- public TaskRunnerHistory getExcutionBlockHistoryByTaskRunnerId(String taskRunnerId) {
- return taskRunnerHistoryMap.get(taskRunnerId);
- }
-
- public TaskRunner getTaskRunner(String taskRunnerId) {
- return taskRunnerMap.get(taskRunnerId);
- }
-
- public Task getTaskByTaskAttemptId(TaskAttemptId taskAttemptId) {
- ExecutionBlockContext context = executionBlockContextMap.get(taskAttemptId.getTaskId().getExecutionBlockId());
- if (context != null) {
- return context.getTask(taskAttemptId);
- }
- return null;
- }
-
- public TaskHistory getTaskHistoryByTaskAttemptId(TaskAttemptId quAttemptId) {
- synchronized (taskRunnerHistoryMap) {
- for (TaskRunnerHistory history : taskRunnerHistoryMap.values()) {
- TaskHistory taskHistory = history.getTaskHistory(quAttemptId);
- if (taskHistory != null) return taskHistory;
- }
- }
-
- return null;
- }
-
- public int getNumTasks() {
- return taskRunnerMap.size();
- }
-
- @Override
- public void handle(TaskRunnerEvent event) {
- LOG.info("======================== Processing " + event.getExecutionBlockId() + " of type " + event.getType());
- if (event instanceof TaskRunnerStartEvent) {
- TaskRunnerStartEvent startEvent = (TaskRunnerStartEvent) event;
- ExecutionBlockContext context = executionBlockContextMap.get(event.getExecutionBlockId());
-
- if(context == null){
- try {
- context = new ExecutionBlockContext(getWorkerContext(), this, startEvent.getRequest());
- context.init();
- } catch (Throwable e) {
- LOG.fatal(e.getMessage(), e);
- throw new RuntimeException(e);
- }
- executionBlockContextMap.put(event.getExecutionBlockId(), context);
- }
-
- TaskRunner taskRunner = new TaskRunner(context, startEvent.getRequest().getContainerId());
- LOG.info("Start TaskRunner:" + taskRunner.getId());
- taskRunnerMap.put(taskRunner.getId(), taskRunner);
- taskRunnerHistoryMap.put(taskRunner.getId(), taskRunner.getHistory());
-
- taskRunner.init(context.getConf());
- taskRunner.start();
-
- } else if (event instanceof TaskRunnerStopEvent) {
- ExecutionBlockContext executionBlockContext = executionBlockContextMap.remove(event.getExecutionBlockId());
- if(executionBlockContext != null){
- try {
- executionBlockContext.getSharedResource().releaseBroadcastCache(event.getExecutionBlockId());
- executionBlockContext.sendShuffleReport();
- workerContext.getTaskHistoryWriter().flushTaskHistories();
- } catch (Exception e) {
- LOG.fatal(e.getMessage(), e);
- throw new RuntimeException(e);
- } finally {
- executionBlockContext.stop();
- }
- }
- LOG.info("Stopped execution block:" + event.getExecutionBlockId());
- }
- }
-
- public EventHandler getEventHandler(){
- return dispatcher.getEventHandler();
- }
-
- public TajoConf getTajoConf() {
- return tajoConf;
- }
-
- class FinishedTaskCleanThread extends Thread {
- //TODO if history size is large, the historyMap should remove immediately
- public void run() {
- int expireIntervalTime = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_HISTORY_EXPIRE_PERIOD);
- LOG.info("FinishedQueryMasterTaskCleanThread started: expire interval minutes = " + expireIntervalTime);
- while(!stop.get()) {
- try {
- Thread.sleep(60 * 1000);
- } catch (InterruptedException e) {
- break;
- }
- try {
- long expireTime = System.currentTimeMillis() - expireIntervalTime * 60 * 1000l;
- cleanExpiredFinishedQueryMasterTask(expireTime);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
- }
-
- private void cleanExpiredFinishedQueryMasterTask(long expireTime) {
- synchronized(taskRunnerHistoryMap) {
- List<String> expiredIds = new ArrayList<String>();
- for(Map.Entry<String, TaskRunnerHistory> entry: taskRunnerHistoryMap.entrySet()) {
- /* If a task runner are abnormal termination, the finished time will be zero. */
- long finishedTime = Math.max(entry.getValue().getStartTime(), entry.getValue().getFinishTime());
- if(finishedTime < expireTime) {
- expiredIds.add(entry.getKey());
- }
- }
-
- for(String eachId: expiredIds) {
- taskRunnerHistoryMap.remove(eachId);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
deleted file mode 100644
index 050e2b5..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.ServerStatusProto;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.TajoHeartbeatResponse;
-import org.apache.tajo.ipc.TajoResourceTrackerProtocol;
-import org.apache.tajo.rpc.CallFuture;
-import org.apache.tajo.rpc.NettyClientBase;
-import org.apache.tajo.rpc.RpcClientManager;
-import org.apache.tajo.service.ServiceTracker;
-import org.apache.tajo.storage.DiskDeviceInfo;
-import org.apache.tajo.storage.DiskMountInfo;
-import org.apache.tajo.storage.DiskUtil;
-
-import java.io.File;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.apache.tajo.ipc.TajoResourceTrackerProtocol.NodeHeartbeat;
-
-/**
- * It periodically sends heartbeat to {@link org.apache.tajo.master.rm.TajoResourceTracker} via asynchronous rpc.
- */
-@Deprecated
-public class WorkerHeartbeatService extends AbstractService {
- /** class logger */
- private final static Log LOG = LogFactory.getLog(WorkerHeartbeatService.class);
-
- private final TajoWorker.WorkerContext context;
- private TajoConf systemConf;
- private RpcClientManager connectionManager;
- private WorkerHeartbeatThread thread;
- private static final float HDFS_DATANODE_STORAGE_SIZE;
-
- static {
- HDFS_DATANODE_STORAGE_SIZE = DiskUtil.getDataNodeStorageSize();
- }
-
- public WorkerHeartbeatService(TajoWorker.WorkerContext context) {
- super(WorkerHeartbeatService.class.getSimpleName());
- this.context = context;
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("Configuration must be a TajoConf instance");
- }
- this.systemConf = (TajoConf) conf;
-
- this.connectionManager = RpcClientManager.getInstance();
- thread = new WorkerHeartbeatThread();
- super.serviceInit(conf);
- }
-
- @Override
- public void serviceStart() throws Exception {
- thread.start();
- super.serviceStart();
- }
-
- @Override
- public void serviceStop() throws Exception {
- if(thread.stopped.getAndSet(true)){
- return;
- }
-
- synchronized (thread) {
- thread.notifyAll();
- }
-
- super.serviceStop();
- }
-
- class WorkerHeartbeatThread extends Thread {
- private volatile AtomicBoolean stopped = new AtomicBoolean(false);
- ServerStatusProto.System systemInfo;
- List<ServerStatusProto.Disk> diskInfos = Lists.newArrayList();
- float workerDiskSlots;
- int workerMemoryMB;
- List<DiskDeviceInfo> diskDeviceInfos;
-
- public WorkerHeartbeatThread() {
- int workerCpuCoreNum;
-
- boolean dedicatedResource = systemConf.getBoolVar(TajoConf.ConfVars.WORKER_RESOURCE_DEDICATED);
-
- try {
- diskDeviceInfos = DiskUtil.getDiskDeviceInfos();
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
-
- if(dedicatedResource) {
- float dedicatedMemoryRatio = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_RESOURCE_DEDICATED_MEMORY_RATIO);
- int totalMemory = getTotalMemoryMB();
- workerMemoryMB = (int) ((float) (totalMemory) * dedicatedMemoryRatio);
- workerCpuCoreNum = Runtime.getRuntime().availableProcessors();
-
- if(diskDeviceInfos == null) {
- workerDiskSlots = TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS.defaultIntVal;
- } else {
- workerDiskSlots = diskDeviceInfos.size();
- }
- } else {
- workerMemoryMB = systemConf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB);
- workerCpuCoreNum = systemConf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES);
- workerDiskSlots = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_DISKS);
-
- if (systemConf.getBoolVar(TajoConf.ConfVars.WORKER_RESOURCE_DFS_DIR_AWARE) && HDFS_DATANODE_STORAGE_SIZE > 0) {
- workerDiskSlots = HDFS_DATANODE_STORAGE_SIZE;
- }
- }
-
- systemInfo = ServerStatusProto.System.newBuilder()
- .setAvailableProcessors(workerCpuCoreNum)
- .setFreeMemoryMB(0)
- .setMaxMemoryMB(0)
- .setTotalMemoryMB(getTotalMemoryMB())
- .build();
- }
-
- public void run() {
- LOG.info("Worker Resource Heartbeat Thread start.");
- int sendDiskInfoCount = 0;
-
- while(!stopped.get()) {
- if(sendDiskInfoCount == 0 && diskDeviceInfos != null) {
- getDiskUsageInfos();
- }
- ServerStatusProto.JvmHeap jvmHeap =
- ServerStatusProto.JvmHeap.newBuilder()
- .setMaxHeap(Runtime.getRuntime().maxMemory())
- .setFreeHeap(Runtime.getRuntime().freeMemory())
- .setTotalHeap(Runtime.getRuntime().totalMemory())
- .build();
-
- ServerStatusProto serverStatus = ServerStatusProto.newBuilder()
- .addAllDisk(diskInfos)
- .setRunningTaskNum(
- context.getTaskRunnerManager() == null ? 1 : context.getTaskRunnerManager().getNumTasks())
- .setSystem(systemInfo)
- .setDiskSlots(workerDiskSlots)
- .setMemoryResourceMB(workerMemoryMB)
- .setJvmHeap(jvmHeap)
- .build();
-
- NodeHeartbeat heartbeatProto = NodeHeartbeat.newBuilder()
- .setConnectionInfo(context.getConnectionInfo().getProto())
- .setServerStatus(serverStatus)
- .build();
-
- NettyClientBase rmClient = null;
- try {
- CallFuture<TajoHeartbeatResponse> callBack = new CallFuture<TajoHeartbeatResponse>();
-
- ServiceTracker serviceTracker = context.getServiceTracker();
- rmClient = connectionManager.getClient(serviceTracker.getResourceTrackerAddress(),
- TajoResourceTrackerProtocol.class, true);
- TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService resourceTracker = rmClient.getStub();
- resourceTracker.heartbeat(callBack.getController(), heartbeatProto, callBack);
-
- TajoHeartbeatResponse response = callBack.get(2, TimeUnit.SECONDS);
-
- QueryCoordinatorProtocol.ClusterResourceSummary clusterResourceSummary = response.getClusterResourceSummary();
- if(clusterResourceSummary.getNumWorkers() > 0) {
- context.setNumClusterNodes(clusterResourceSummary.getNumWorkers());
- }
- context.setClusterResource(clusterResourceSummary);
-
- } catch (InterruptedException e) {
- break;
- } catch (TimeoutException te) {
- LOG.warn("Heartbeat response is being delayed.", te);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
-
- try {
- if(!stopped.get()){
- synchronized (thread){
- thread.wait(10 * 1000);
- }
- }
- } catch (InterruptedException e) {
- break;
- }
- sendDiskInfoCount++;
-
- if(sendDiskInfoCount > 10) {
- sendDiskInfoCount = 0;
- }
- }
-
- LOG.info("Worker Resource Heartbeat Thread stopped.");
- }
-
- private void getDiskUsageInfos() {
- diskInfos.clear();
- for(DiskDeviceInfo eachDevice: diskDeviceInfos) {
- List<DiskMountInfo> mountInfos = eachDevice.getMountInfos();
- if(mountInfos != null) {
- for(DiskMountInfo eachMount: mountInfos) {
- File eachFile = new File(eachMount.getMountPath());
- diskInfos.add(ServerStatusProto.Disk.newBuilder()
- .setAbsolutePath(eachFile.getAbsolutePath())
- .setTotalSpace(eachFile.getTotalSpace())
- .setFreeSpace(eachFile.getFreeSpace())
- .setUsableSpace(eachFile.getUsableSpace())
- .build());
- }
- }
- }
- }
- }
-
- public static int getTotalMemoryMB() {
- javax.management.MBeanServer mBeanServer = java.lang.management.ManagementFactory.getPlatformMBeanServer();
- long max = 0;
- Object maxObject = null;
- try {
- javax.management.ObjectName osName = new javax.management.ObjectName("java.lang:type=OperatingSystem");
- if (!System.getProperty("java.vendor").startsWith("IBM")) {
- maxObject = mBeanServer.getAttribute(osName, "TotalPhysicalMemorySize");
- } else {
- maxObject = mBeanServer.getAttribute(osName, "TotalPhysicalMemory");
- }
- } catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- }
- if (maxObject != null) {
- max = ((Long)maxObject).longValue();
- }
- return ((int) (max / (1024 * 1024)));
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockErrorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockErrorEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockErrorEvent.java
new file mode 100644
index 0000000..dfc54ab
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockErrorEvent.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.worker.event;
+
+import org.apache.tajo.ExecutionBlockId;
+
+public class ExecutionBlockErrorEvent extends TaskManagerEvent {
+
+ private ExecutionBlockId executionBlockId;
+ private Throwable error;
+
+ public ExecutionBlockErrorEvent(ExecutionBlockId executionBlockId, Throwable e) {
+ super(EventType.EB_FAIL);
+ this.executionBlockId = executionBlockId;
+ this.error = e;
+ }
+
+ public ExecutionBlockId getExecutionBlockId() {
+ return executionBlockId;
+ }
+
+ public Throwable getError() {
+ return error;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStartEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStartEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStartEvent.java
deleted file mode 100644
index 85d74e2..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStartEvent.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker.event;
-
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-
-public class ExecutionBlockStartEvent extends TaskManagerEvent {
- private TajoWorkerProtocol.RunExecutionBlockRequestProto requestProto;
-
- public ExecutionBlockStartEvent(TajoWorkerProtocol.RunExecutionBlockRequestProto requestProto) {
- super(EventType.EB_START, new ExecutionBlockId(requestProto.getExecutionBlockId()));
- this.requestProto = requestProto;
- }
-
- public TajoWorkerProtocol.RunExecutionBlockRequestProto getRequestProto() {
- return requestProto;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStopEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStopEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStopEvent.java
index 2b967ab..a1dfe50 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStopEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/ExecutionBlockStopEvent.java
@@ -20,18 +20,24 @@ package org.apache.tajo.worker.event;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.TajoIdProtos;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ResourceProtos.ExecutionBlockListProto;
public class ExecutionBlockStopEvent extends TaskManagerEvent {
- private TajoWorkerProtocol.ExecutionBlockListProto cleanupList;
+ private ExecutionBlockListProto cleanupList;
+ private ExecutionBlockId executionBlockId;
public ExecutionBlockStopEvent(TajoIdProtos.ExecutionBlockIdProto executionBlockId,
- TajoWorkerProtocol.ExecutionBlockListProto cleanupList) {
- super(EventType.EB_STOP, new ExecutionBlockId(executionBlockId));
+ ExecutionBlockListProto cleanupList) {
+ super(EventType.EB_STOP);
this.cleanupList = cleanupList;
+ this.executionBlockId = new ExecutionBlockId(executionBlockId);
}
- public TajoWorkerProtocol.ExecutionBlockListProto getCleanupList() {
+ public ExecutionBlockListProto getCleanupList() {
return cleanupList;
}
+
+ public ExecutionBlockId getExecutionBlockId() {
+ return executionBlockId;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceAllocateEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceAllocateEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceAllocateEvent.java
index 9a1c106..0ee0836 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceAllocateEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceAllocateEvent.java
@@ -21,26 +21,26 @@ package org.apache.tajo.worker.event;
import com.google.protobuf.RpcCallback;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.BatchAllocationRequestProto;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.BatchAllocationResponseProto;
+import static org.apache.tajo.ResourceProtos.BatchAllocationRequest;
+import static org.apache.tajo.ResourceProtos.BatchAllocationResponse;
public class NodeResourceAllocateEvent extends NodeResourceEvent {
- private BatchAllocationRequestProto request;
- private RpcCallback<BatchAllocationResponseProto> callback;
+ private BatchAllocationRequest request;
+ private RpcCallback<BatchAllocationResponse> callback;
- public NodeResourceAllocateEvent(BatchAllocationRequestProto request,
- RpcCallback<BatchAllocationResponseProto> callback) {
- super(EventType.ALLOCATE);
+ public NodeResourceAllocateEvent(BatchAllocationRequest request,
+ RpcCallback<BatchAllocationResponse> callback) {
+ super(EventType.ALLOCATE, ResourceType.TASK);
this.callback = callback;
this.request = request;
}
- public BatchAllocationRequestProto getRequest() {
+ public BatchAllocationRequest getRequest() {
return request;
}
- public RpcCallback<BatchAllocationResponseProto> getCallback() {
+ public RpcCallback<BatchAllocationResponse> getCallback() {
return callback;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceDeallocateEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceDeallocateEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceDeallocateEvent.java
index 31d9229..d8841a2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceDeallocateEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceDeallocateEvent.java
@@ -25,12 +25,12 @@ public class NodeResourceDeallocateEvent extends NodeResourceEvent {
private NodeResource resource;
- public NodeResourceDeallocateEvent(TajoProtos.NodeResourceProto proto) {
- this(new NodeResource(proto));
+ public NodeResourceDeallocateEvent(TajoProtos.NodeResourceProto proto, ResourceType resourceType) {
+ this(new NodeResource(proto), resourceType);
}
- public NodeResourceDeallocateEvent(NodeResource resource) {
- super(EventType.DEALLOCATE);
+ public NodeResourceDeallocateEvent(NodeResource resource, ResourceType resourceType) {
+ super(EventType.DEALLOCATE, resourceType);
this.resource = resource;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceEvent.java
index 6fd2e0d..c12551f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/NodeResourceEvent.java
@@ -29,7 +29,19 @@ public class NodeResourceEvent extends AbstractEvent<NodeResourceEvent.EventType
DEALLOCATE
}
- public NodeResourceEvent(EventType eventType) {
+ public enum ResourceType {
+ QUERY_MASTER,
+ TASK
+ }
+
+ private ResourceType resourceType;
+
+ public NodeResourceEvent(EventType eventType, ResourceType resourceType) {
super(eventType);
+ this.resourceType = resourceType;
+ }
+
+ public ResourceType getResourceType() {
+ return resourceType;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/QMResourceAllocateEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/QMResourceAllocateEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/QMResourceAllocateEvent.java
new file mode 100644
index 0000000..4422d4d
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/QMResourceAllocateEvent.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.worker.event;
+
+
+import com.google.protobuf.RpcCallback;
+import org.apache.tajo.ResourceProtos.AllocationResourceProto;
+import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
+
+public class QMResourceAllocateEvent extends NodeResourceEvent {
+
+ private AllocationResourceProto request;
+ private RpcCallback<PrimitiveProtos.BoolProto> callback;
+
+ public QMResourceAllocateEvent(AllocationResourceProto request,
+ RpcCallback<PrimitiveProtos.BoolProto> callback) {
+ super(EventType.ALLOCATE, ResourceType.QUERY_MASTER);
+ this.callback = callback;
+ this.request = request;
+ }
+
+ public AllocationResourceProto getRequest() {
+ return request;
+ }
+
+ public RpcCallback<PrimitiveProtos.BoolProto> getCallback() {
+ return callback;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/QueryStopEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/QueryStopEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/QueryStopEvent.java
new file mode 100644
index 0000000..892db92
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/QueryStopEvent.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.worker.event;
+
+import org.apache.tajo.QueryId;
+
+public class QueryStopEvent extends TaskManagerEvent {
+
+
+ private QueryId queryId;
+ public QueryStopEvent(QueryId queryId) {
+ super(EventType.QUERY_STOP);
+ this.queryId = queryId;
+ }
+
+ public QueryId getQueryId() {
+ return queryId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskExecutorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskExecutorEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskExecutorEvent.java
deleted file mode 100644
index c609c67..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskExecutorEvent.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker.event;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.TaskAttemptId;
-
-public class TaskExecutorEvent extends AbstractEvent<TaskExecutorEvent.EventType> {
-
- // producer: NodeResourceManager, consumer: TaskExecutorEvent
- public enum EventType {
- START,
- KILL,
- ABORT
- }
-
- private TaskAttemptId taskAttemptId;
-
- public TaskExecutorEvent(EventType eventType,
- TaskAttemptId taskAttemptId) {
- super(eventType);
- this.taskAttemptId = taskAttemptId;
- }
-
- public TaskAttemptId getTaskId() {
- return taskAttemptId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskManagerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskManagerEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskManagerEvent.java
index 39b541b..7225e70 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskManagerEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskManagerEvent.java
@@ -19,25 +19,23 @@
package org.apache.tajo.worker.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.TaskAttemptId;
public class TaskManagerEvent extends AbstractEvent<TaskManagerEvent.EventType> {
// producer: NodeResourceManager, consumer: TaskManager
public enum EventType {
- EB_START,
- EB_STOP
+ TASK_START,
+ TASK_KILL,
+ TASK_ABORT,
+
+ //cleanup events
+ EB_STOP,
+ EB_FAIL,
+ QUERY_STOP
}
- private ExecutionBlockId executionBlockId;
- public TaskManagerEvent(EventType eventType,
- ExecutionBlockId executionBlockId) {
- super(eventType);
- this.executionBlockId = executionBlockId;
- }
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
+ public TaskManagerEvent(EventType eventType) {
+ super(eventType);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerEvent.java
deleted file mode 100644
index 7175251..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerEvent.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker.event;
-
-import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.ExecutionBlockId;
-
-@Deprecated
-public class TaskRunnerEvent extends AbstractEvent<TaskRunnerEvent.EventType> {
- public enum EventType {
- START,
- STOP
- }
-
- protected final ExecutionBlockId executionBlockId;
-
- public TaskRunnerEvent(EventType eventType,
- ExecutionBlockId executionBlockId) {
- super(eventType);
- this.executionBlockId = executionBlockId;
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStartEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStartEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStartEvent.java
deleted file mode 100644
index 9406794..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStartEvent.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker.event;
-
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.plan.serder.PlanProto;
-@Deprecated
-public class TaskRunnerStartEvent extends TaskRunnerEvent {
-
- private final TajoWorkerProtocol.RunExecutionBlockRequestProto request;
-
- public TaskRunnerStartEvent(TajoWorkerProtocol.RunExecutionBlockRequestProto request) {
- super(EventType.START, new ExecutionBlockId(request.getExecutionBlockId()));
- this.request = request;
- }
-
- public TajoWorkerProtocol.RunExecutionBlockRequestProto getRequest() {
- return request;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStopEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStopEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStopEvent.java
deleted file mode 100644
index 297f30c..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskRunnerStopEvent.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker.event;
-
-import org.apache.tajo.ExecutionBlockId;
-
-@Deprecated
-public class TaskRunnerStopEvent extends TaskRunnerEvent {
-
- public TaskRunnerStopEvent(ExecutionBlockId executionBlockId) {
- super(EventType.STOP, executionBlockId);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskStartEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskStartEvent.java b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskStartEvent.java
index f60e7c4..1fb0c49 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskStartEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/event/TaskStartEvent.java
@@ -18,20 +18,24 @@
package org.apache.tajo.worker.event;
+import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.resource.NodeResource;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto;
-public class TaskStartEvent extends TaskExecutorEvent {
+import static org.apache.tajo.ResourceProtos.TaskRequestProto;
+
+public class TaskStartEvent extends TaskManagerEvent {
private NodeResource allocatedResource;
private TaskRequestProto taskRequest;
+ private TaskAttemptId taskAttemptId;
public TaskStartEvent(TaskRequestProto taskRequest,
NodeResource allocatedResource) {
- super(EventType.START, new TaskAttemptId(taskRequest.getId()));
+ super(EventType.TASK_START);
this.taskRequest = taskRequest;
this.allocatedResource = allocatedResource;
+ this.taskAttemptId = new TaskAttemptId(taskRequest.getId());
}
public NodeResource getAllocatedResource() {
@@ -41,4 +45,12 @@ public class TaskStartEvent extends TaskExecutorEvent {
public TaskRequestProto getTaskRequest() {
return taskRequest;
}
+
+ public TaskAttemptId getTaskAttemptId() {
+ return taskAttemptId;
+ }
+
+ public ExecutionBlockId getExecutionBlockId() {
+ return taskAttemptId.getTaskId().getExecutionBlockId();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java
index 349fa4c..19d8b28 100644
--- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java
+++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java
@@ -35,7 +35,7 @@ import javax.ws.rs.core.UriInfo;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.master.TajoMaster.MasterContext;
-import org.apache.tajo.master.rm.Worker;
+import org.apache.tajo.master.rm.NodeStatus;
import org.apache.tajo.ws.rs.JerseyResourceDelegate;
import org.apache.tajo.ws.rs.JerseyResourceDelegateContext;
import org.apache.tajo.ws.rs.JerseyResourceDelegateContextKey;
@@ -99,11 +99,11 @@ public class ClusterResource {
JerseyResourceDelegateContextKey.valueOf(JerseyResourceDelegateUtil.MasterContextKey, MasterContext.class);
MasterContext masterContext = context.get(masterContextKey);
- Map<Integer, Worker> workerMap = masterContext.getResourceManager().getWorkers();
+ Map<Integer, NodeStatus> workerMap = masterContext.getResourceManager().getNodes();
List<WorkerResponse> workerList = new ArrayList<WorkerResponse>();
- for (Worker worker: workerMap.values()) {
- workerList.add(new WorkerResponse(worker));
+ for (NodeStatus nodeStatus : workerMap.values()) {
+ workerList.add(new WorkerResponse(nodeStatus));
}
Map<String, List<WorkerResponse>> workerResponseMap = new HashMap<String, List<WorkerResponse>>();
[06/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index 1f5e7a3..61e2dea 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@ -18,6 +18,8 @@
package org.apache.tajo.querymaster;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -35,11 +37,9 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.exception.UnimplementedException;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.TajoContainerProxy;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.rm.TajoWorkerResourceManager;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
@@ -48,20 +48,29 @@ import org.apache.tajo.plan.logical.LogicalRootNode;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.resource.NodeResource;
+import org.apache.tajo.rpc.*;
+import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.session.Session;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.FormatProperty;
+import org.apache.tajo.storage.Tablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.util.metrics.TajoMetrics;
import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter;
-import org.apache.tajo.worker.AbstractResourceAllocator;
-import org.apache.tajo.worker.TajoResourceAllocator;
+import org.apache.tajo.worker.event.NodeResourceDeallocateEvent;
+import org.apache.tajo.worker.event.NodeResourceEvent;
+import org.apache.tajo.worker.event.NodeStatusEvent;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.tajo.TajoProtos.QueryState;
+import static org.apache.tajo.ResourceProtos.*;
public class QueryMasterTask extends CompositeService {
private static final Log LOG = LogFactory.getLog(QueryMasterTask.class.getName());
@@ -90,20 +99,21 @@ public class QueryMasterTask extends CompositeService {
private AtomicLong lastClientHeartbeat = new AtomicLong(-1);
- private AbstractResourceAllocator resourceAllocator;
-
- private AtomicBoolean stopped = new AtomicBoolean(false);
+ private volatile boolean isStopped;
private TajoMetrics queryMetrics;
private Throwable initError;
- private final List<TajoWorkerProtocol.TaskFatalErrorReport> diagnostics =
- new ArrayList<TajoWorkerProtocol.TaskFatalErrorReport>();
+ private NodeResource allocation;
+
+ private final List<TaskFatalErrorReport> diagnostics = new ArrayList<TaskFatalErrorReport>();
+
+ private final ConcurrentMap<Integer, WorkerConnectionInfo> workerMap = Maps.newConcurrentMap();
public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext,
QueryId queryId, Session session, QueryContext queryContext,
- String jsonExpr, AsyncDispatcher dispatcher) {
+ String jsonExpr, NodeResource allocation, AsyncDispatcher dispatcher) {
super(QueryMasterTask.class.getName());
this.queryMasterContext = queryMasterContext;
@@ -111,90 +121,90 @@ public class QueryMasterTask extends CompositeService {
this.session = session;
this.queryContext = queryContext;
this.jsonExpr = jsonExpr;
+ this.allocation = allocation;
this.querySubmitTime = System.currentTimeMillis();
this.dispatcher = dispatcher;
}
public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext,
QueryId queryId, Session session, QueryContext queryContext,
- String jsonExpr) {
- this(queryMasterContext, queryId, session, queryContext, jsonExpr, new AsyncDispatcher());
+ String jsonExpr,
+ NodeResource allocation) {
+ this(queryMasterContext, queryId, session, queryContext, jsonExpr, allocation, new AsyncDispatcher());
}
@Override
- public void init(Configuration conf) {
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("conf should be a TajoConf type.");
- }
- systemConf = (TajoConf)conf;
+ public void serviceInit(Configuration conf) throws Exception {
- try {
- queryTaskContext = new QueryMasterTaskContext();
- String resourceManagerClassName = systemConf.getVar(TajoConf.ConfVars.RESOURCE_MANAGER_CLASS);
+ systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
- if(resourceManagerClassName.indexOf(TajoWorkerResourceManager.class.getName()) >= 0) {
- resourceAllocator = new TajoResourceAllocator(queryTaskContext);
- } else {
- throw new UnimplementedException(resourceManagerClassName + " is not supported yet");
- }
- addService(resourceAllocator);
- addService(dispatcher);
+ queryTaskContext = new QueryMasterTaskContext();
- dispatcher.register(StageEventType.class, new StageEventDispatcher());
- dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
- dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
- dispatcher.register(QueryMasterQueryCompletedEvent.EventType.class, new QueryFinishEventHandler());
- dispatcher.register(TaskSchedulerEvent.EventType.class, new TaskSchedulerDispatcher());
- dispatcher.register(LocalTaskEventType.class, new LocalTaskEventHandler());
+ addService(dispatcher);
- queryMetrics = new TajoMetrics(queryId.toString());
+ dispatcher.register(StageEventType.class, new StageEventDispatcher());
+ dispatcher.register(TaskEventType.class, new TaskEventDispatcher());
+ dispatcher.register(TaskAttemptEventType.class, new TaskAttemptEventDispatcher());
+ dispatcher.register(QueryMasterQueryCompletedEvent.EventType.class, new QueryFinishEventHandler());
+ dispatcher.register(TaskSchedulerEvent.EventType.class, new TaskSchedulerDispatcher());
+ dispatcher.register(LocalTaskEventType.class, new LocalTaskEventHandler());
- super.init(systemConf);
+ try {
+ queryMetrics = new TajoMetrics(queryId.toString());
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
initError = t;
}
+ super.serviceInit(systemConf);
}
public boolean isStopped() {
- return stopped.get();
+ return isStopped;
}
@Override
- public void start() {
+ public void serviceStart() throws Exception {
startQuery();
- super.start();
+ List<TajoProtos.WorkerConnectionInfoProto> workersProto = queryMasterContext.getQueryMaster().getAllWorker();
+ for (TajoProtos.WorkerConnectionInfoProto worker : workersProto) {
+ workerMap.put(worker.getId(), new WorkerConnectionInfo(worker));
+ }
+ super.serviceStart();
}
@Override
- public void stop() {
-
- if(stopped.getAndSet(true)) {
- return;
- }
+ public void serviceStop() throws Exception {
+ isStopped = true;
LOG.info("Stopping QueryMasterTask:" + queryId);
- try {
- resourceAllocator.stop();
- } catch (Throwable t) {
- LOG.fatal(t.getMessage(), t);
+ //release QM resource
+ EventHandler handler = getQueryTaskContext().getQueryMasterContext().getWorkerContext().
+ getNodeResourceManager().getDispatcher().getEventHandler();
+
+ handler.handle(new NodeResourceDeallocateEvent(allocation, NodeResourceEvent.ResourceType.QUERY_MASTER));
+
+ //flush current node resource
+ handler.handle(new NodeStatusEvent(NodeStatusEvent.EventType.FLUSH_REPORTS));
+
+ if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) {
+ cleanupQuery(getQueryId());
}
if (queryMetrics != null) {
- queryMetrics.report(new MetricsConsoleReporter());
+ queryMasterContext.getEventExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ queryMetrics.report(new MetricsConsoleReporter());
+ }
+ });
}
- super.stop();
+ super.serviceStop();
LOG.info("Stopped QueryMasterTask:" + queryId);
}
- public void handleTaskRequestEvent(TaskRequestEvent event) {
- ExecutionBlockId id = event.getExecutionBlockId();
- query.getStage(id).handleTaskRequestEvent(event);
- }
-
- public void handleTaskFailed(TajoWorkerProtocol.TaskFatalErrorReport report) {
+ public void handleTaskFailed(TaskFatalErrorReport report) {
synchronized(diagnostics) {
if (diagnostics.size() < 10) {
diagnostics.add(report);
@@ -204,7 +214,7 @@ public class QueryMasterTask extends CompositeService {
getEventHandler().handle(new TaskFatalErrorEvent(report));
}
- public Collection<TajoWorkerProtocol.TaskFatalErrorReport> getDiagnostics() {
+ public Collection<TaskFatalErrorReport> getDiagnostics() {
synchronized(diagnostics) {
return Collections.unmodifiableCollection(diagnostics);
}
@@ -252,33 +262,53 @@ public class QueryMasterTask extends CompositeService {
}
}
+ /**
+ * It sends a kill RPC request to a corresponding worker.
+ *
+ * @param workerId worker unique Id.
+ * @param taskAttemptId The TaskAttemptId to be killed.
+ */
+ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) {
+ NettyClientBase tajoWorkerRpc;
+ ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId();
+ InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId);
+
+ try {
+ tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
+ CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>();
+ tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture);
+
+ if(!callFuture.get().getValue()){
+ queryMasterContext.getEventHandler().handle(
+ new TaskFatalErrorEvent(taskAttemptId, "Can't kill task :" + taskAttemptId));
+ }
+ } catch (Exception e) {
+ /* Node RPC failure */
+ LOG.error(e.getMessage(), e);
+ queryMasterContext.getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e.getMessage()));
+ }
+ }
+
private class LocalTaskEventHandler implements EventHandler<LocalTaskEvent> {
@Override
- public void handle(LocalTaskEvent event) {
- TajoContainerProxy proxy = (TajoContainerProxy) resourceAllocator.getContainers().get(event.getContainerId());
- if (proxy != null) {
- proxy.killTaskAttempt(event.getTaskAttemptId());
- }
+ public void handle(final LocalTaskEvent event) {
+ queryMasterContext.getEventExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ killTaskAttempt(event.getWorkerId(), event.getTaskAttemptId());
+ }
+ });
}
}
private class QueryFinishEventHandler implements EventHandler<QueryMasterQueryCompletedEvent> {
+
@Override
public void handle(QueryMasterQueryCompletedEvent event) {
- QueryId queryId = event.getQueryId();
- LOG.info("Query completion notified from " + queryId);
-
- while (!isTerminatedState(query.getSynchronizedState())) {
- try {
- synchronized (this) {
- wait(10);
- }
- } catch (InterruptedException e) {
- LOG.error(e);
- }
- }
- LOG.info("Query final state: " + query.getSynchronizedState());
+ QueryId queryId = event.getQueryId();
+ LOG.info("Query completion notified from " + queryId + " final state: " + query.getSynchronizedState());
queryMasterContext.getEventHandler().handle(new QueryStopEvent(queryId));
}
}
@@ -441,6 +471,29 @@ public class QueryMasterTask extends CompositeService {
return this.querySubmitTime;
}
+ private void cleanupQuery(final QueryId queryId) {
+ Set<InetSocketAddress> workers = Sets.newHashSet();
+ for (Stage stage : getQuery().getStages()) {
+ workers.addAll(stage.getAssignedWorkerMap().values());
+ }
+
+ LOG.info("Cleanup resources of all workers. Query: " + queryId + ", workers: " + workers.size());
+ for (final InetSocketAddress worker : workers) {
+ queryMasterContext.getEventExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub();
+ tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get());
+ } catch (Throwable e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ });
+ }
+ }
+
public class QueryMasterTaskContext {
EventHandler eventHandler;
public QueryMaster.QueryMasterContext getQueryMasterContext() {
@@ -501,12 +554,15 @@ public class QueryMasterTask extends CompositeService {
return query.getProgress();
}
- public AbstractResourceAllocator getResourceAllocator() {
- return resourceAllocator;
- }
-
public TajoMetrics getQueryMetrics() {
return queryMetrics;
}
+
+ /**
+ * A key is worker id, and a value is a worker connection information.
+ */
+ public ConcurrentMap<Integer, WorkerConnectionInfo> getWorkerMap() {
+ return workerMap;
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index f30fb64..50b84d4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -40,8 +40,8 @@ import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.planner.global.rewriter.rules.GlobalPlanRewriteUtil;
import org.apache.tajo.engine.utils.TupleUtil;
import org.apache.tajo.exception.InternalException;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
import org.apache.tajo.querymaster.Task.IntermediateEntry;
import org.apache.tajo.plan.logical.SortNode.SortPurpose;
import org.apache.tajo.plan.util.PlannerUtil;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 1163a6e..50b45a8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -20,15 +20,12 @@ package org.apache.tajo.querymaster;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.tajo.*;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
@@ -38,27 +35,24 @@ import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.StatisticsUtil;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
-import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty;
-import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto;
-import org.apache.tajo.master.LaunchTaskRunnersEvent;
-import org.apache.tajo.master.TaskRunnerGroupEvent;
-import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
+import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage;
+import org.apache.tajo.plan.serder.PlanProto.EnforceProperty;
import org.apache.tajo.master.TaskState;
-import org.apache.tajo.master.container.TajoContainer;
-import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.querymaster.Task.IntermediateEntry;
+import org.apache.tajo.rpc.AsyncRpcClient;
+import org.apache.tajo.rpc.NullCallback;
+import org.apache.tajo.rpc.RpcClientManager;
+import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.storage.FileTablespace;
import org.apache.tajo.storage.TablespaceManager;
import org.apache.tajo.storage.Tablespace;
@@ -70,8 +64,8 @@ import org.apache.tajo.util.history.TaskHistory;
import org.apache.tajo.worker.FetchImpl;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
@@ -80,6 +74,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.tajo.conf.TajoConf.ConfVars;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
+import static org.apache.tajo.ResourceProtos.*;
/**
@@ -107,16 +102,12 @@ public class Stage implements EventHandler<StageEvent> {
private volatile long lastContactTime;
private Thread timeoutChecker;
- volatile Map<TaskId, Task> tasks = new ConcurrentHashMap<TaskId, Task>();
- volatile Map<TajoContainerId, TajoContainer> containers = new ConcurrentHashMap<TajoContainerId,
- TajoContainer>();
+ private final Map<TaskId, Task> tasks = Maps.newConcurrentMap();
+ private final Map<Integer, InetSocketAddress> workerMap = Maps.newConcurrentMap();
private static final DiagnosticsUpdateTransition DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition();
private static final InternalErrorTransition INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
- private static final ContainerLaunchTransition CONTAINER_LAUNCH_TRANSITION = new ContainerLaunchTransition();
private static final TaskCompletedTransition TASK_COMPLETED_TRANSITION = new TaskCompletedTransition();
- private static final AllocatedContainersCancelTransition CONTAINERS_CANCEL_TRANSITION =
- new AllocatedContainersCancelTransition();
private static final StageCompleteTransition STAGE_COMPLETED_TRANSITION = new StageCompleteTransition();
private static final StageFinalizeTransition STAGE_FINALIZE_TRANSITION = new StageFinalizeTransition();
private StateMachine<StageState, StageEventType, StageEvent> stateMachine;
@@ -142,8 +133,7 @@ public class Stage implements EventHandler<StageEvent> {
// Transitions from INITED state
.addTransition(StageState.INITED, StageState.RUNNING,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINER_LAUNCH_TRANSITION)
+ StageEventType.SQ_START)
.addTransition(StageState.INITED, StageState.INITED,
StageEventType.SQ_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
@@ -153,10 +143,7 @@ public class Stage implements EventHandler<StageEvent> {
StageEventType.SQ_INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
- // Transitions from RUNNING state
- .addTransition(StageState.RUNNING, StageState.RUNNING,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINER_LAUNCH_TRANSITION)
+ // Transitions from RUNNING state
.addTransition(StageState.RUNNING, StageState.RUNNING,
StageEventType.SQ_TASK_COMPLETED,
TASK_COMPLETED_TRANSITION)
@@ -185,10 +172,8 @@ public class Stage implements EventHandler<StageEvent> {
// Transitions from KILL_WAIT state
.addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
- EnumSet.of(StageEventType.SQ_START, StageEventType.SQ_CONTAINER_ALLOCATED),
- CONTAINERS_CANCEL_TRANSITION)
- .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
- EnumSet.of(StageEventType.SQ_KILL), new KillTasksTransition())
+ EnumSet.of(StageEventType.SQ_KILL),
+ new KillTasksTransition())
.addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
StageEventType.SQ_TASK_COMPLETED,
TASK_COMPLETED_TRANSITION)
@@ -205,6 +190,9 @@ public class Stage implements EventHandler<StageEvent> {
.addTransition(StageState.KILL_WAIT, StageState.ERROR,
StageEventType.SQ_INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
+ // Ignore-able events
+ .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
+ EnumSet.of(StageEventType.SQ_START))
// Transitions from FINALIZING state
.addTransition(StageState.FINALIZING, StageState.FINALIZING,
@@ -226,9 +214,6 @@ public class Stage implements EventHandler<StageEvent> {
// Transitions from SUCCEEDED state
.addTransition(StageState.SUCCEEDED, StageState.SUCCEEDED,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINERS_CANCEL_TRANSITION)
- .addTransition(StageState.SUCCEEDED, StageState.SUCCEEDED,
StageEventType.SQ_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(StageState.SUCCEEDED, StageState.ERROR,
@@ -239,14 +224,10 @@ public class Stage implements EventHandler<StageEvent> {
EnumSet.of(
StageEventType.SQ_START,
StageEventType.SQ_KILL,
- StageEventType.SQ_CONTAINER_ALLOCATED,
StageEventType.SQ_SHUFFLE_REPORT))
// Transitions from KILLED state
.addTransition(StageState.KILLED, StageState.KILLED,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINERS_CANCEL_TRANSITION)
- .addTransition(StageState.KILLED, StageState.KILLED,
StageEventType.SQ_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(StageState.KILLED, StageState.ERROR,
@@ -257,16 +238,12 @@ public class Stage implements EventHandler<StageEvent> {
EnumSet.of(
StageEventType.SQ_START,
StageEventType.SQ_KILL,
- StageEventType.SQ_CONTAINER_ALLOCATED,
StageEventType.SQ_SHUFFLE_REPORT,
StageEventType.SQ_STAGE_COMPLETED,
StageEventType.SQ_FAILED))
// Transitions from FAILED state
.addTransition(StageState.FAILED, StageState.FAILED,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINERS_CANCEL_TRANSITION)
- .addTransition(StageState.FAILED, StageState.FAILED,
StageEventType.SQ_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(StageState.FAILED, StageState.ERROR,
@@ -277,14 +254,10 @@ public class Stage implements EventHandler<StageEvent> {
EnumSet.of(
StageEventType.SQ_START,
StageEventType.SQ_KILL,
- StageEventType.SQ_CONTAINER_ALLOCATED,
StageEventType.SQ_FAILED))
// Transitions from ERROR state
.addTransition(StageState.ERROR, StageState.ERROR,
- StageEventType.SQ_CONTAINER_ALLOCATED,
- CONTAINERS_CANCEL_TRANSITION)
- .addTransition(StageState.ERROR, StageState.ERROR,
StageEventType.SQ_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
// Ignore-able transitions
@@ -294,18 +267,19 @@ public class Stage implements EventHandler<StageEvent> {
StageEventType.SQ_KILL,
StageEventType.SQ_FAILED,
StageEventType.SQ_INTERNAL_ERROR,
- StageEventType.SQ_STAGE_COMPLETED))
+ StageEventType.SQ_STAGE_COMPLETED,
+ StageEventType.SQ_SHUFFLE_REPORT))
.installTopology();
private final Lock readLock;
private final Lock writeLock;
- private int totalScheduledObjectsCount;
- private int completedTaskCount = 0;
- private int succeededObjectCount = 0;
- private int killedObjectCount = 0;
- private int failedObjectCount = 0;
+ private volatile int totalScheduledObjectsCount;
+ private volatile int completedTaskCount = 0;
+ private volatile int succeededObjectCount = 0;
+ private volatile int killedObjectCount = 0;
+ private volatile int failedObjectCount = 0;
private TaskSchedulerContext schedulerContext;
private List<IntermediateEntry> hashShuffleIntermediateEntries = Lists.newArrayList();
private AtomicInteger completedShuffleTasks = new AtomicInteger(0);
@@ -695,16 +669,60 @@ public class Stage implements EventHandler<StageEvent> {
}
private void stopScheduler() {
- // If there are launched TaskRunners, send the 'shouldDie' message to all r
- // via received task requests.
if (taskScheduler != null) {
taskScheduler.stop();
}
}
- private void releaseContainers() {
- // If there are still live TaskRunners, try to kill the containers. and send the shuffle report request
- eventHandler.handle(new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_CLEANUP, getId(), containers.values()));
+ /**
+ * Get the launched worker address
+ */
+ protected Map<Integer, InetSocketAddress> getAssignedWorkerMap() {
+ return workerMap;
+ }
+
+ private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest requestProto) {
+
+ for (final InetSocketAddress worker : getAssignedWorkerMap().values()) {
+ getContext().getQueryMasterContext().getEventExecutor().submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ AsyncRpcClient tajoWorkerRpc =
+ RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
+ tajoWorkerRpcClient.stopExecutionBlock(null,
+ requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class));
+ } catch (Throwable e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ });
+ }
+ }
+
+ /**
+ * Sends stopping request to all worker
+ */
+ protected void stopExecutionBlock() {
+ // If there are still live tasks, try to kill the tasks. and send the shuffle report request
+
+ List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList();
+ if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) {
+ List<ExecutionBlock> childs = getMasterPlan().getChilds(getId());
+
+ for (ExecutionBlock executionBlock : childs) {
+ ebIds.add(executionBlock.getId().getProto());
+ }
+ }
+
+ StopExecutionBlockRequest.Builder stopRequest = StopExecutionBlockRequest.newBuilder();
+ ExecutionBlockListProto.Builder cleanupList = ExecutionBlockListProto.newBuilder();
+
+ cleanupList.addAllExecutionBlockId(Lists.newArrayList(ebIds));
+ stopRequest.setCleanupList(cleanupList.build());
+ stopRequest.setExecutionBlockId(getId().getProto());
+ sendStopExecutionBlockEvent(stopRequest.build());
}
/**
@@ -769,10 +787,6 @@ public class Stage implements EventHandler<StageEvent> {
}
}
- public void handleTaskRequestEvent(TaskRequestEvent event) {
- taskScheduler.handleTaskRequestEvent(event);
- }
-
private static class InitAndRequestContainer implements MultipleArcTransition<Stage,
StageEvent, StageState> {
@@ -794,7 +808,7 @@ public class Stage implements EventHandler<StageEvent> {
setShuffleIfNecessary(stage, channel);
initTaskScheduler(stage);
// execute pre-processing asyncronously
- stage.getContext().getQueryMasterContext().getEventExecutor()
+ stage.getContext().getQueryMasterContext().getSingleEventExecutor()
.submit(new Runnable() {
@Override
public void run() {
@@ -809,7 +823,7 @@ public class Stage implements EventHandler<StageEvent> {
} else {
if(stage.getSynchronizedState() == StageState.INITED) {
stage.taskScheduler.start();
- allocateContainers(stage);
+ stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_START));
} else {
/* all tasks are killed before stage are inited */
if (stage.getTotalScheduledObjectsCount() == stage.getCompletedTaskCount()) {
@@ -1047,33 +1061,6 @@ public class Stage implements EventHandler<StageEvent> {
}
}
- public static void allocateContainers(Stage stage) {
- ExecutionBlock execBlock = stage.getBlock();
-
- //TODO consider disk slot
- int requiredMemoryMBPerTask = 512;
-
- int numRequest = stage.getContext().getResourceAllocator().calculateNumRequestContainers(
- stage.getContext().getQueryMasterContext().getWorkerContext(),
- stage.schedulerContext.getEstimatedTaskNum(),
- requiredMemoryMBPerTask
- );
-
- final Resource resource = Records.newRecord(Resource.class);
-
- resource.setMemory(requiredMemoryMBPerTask);
-
- LOG.info("Request Container for " + stage.getId() + " containers=" + numRequest);
-
- Priority priority = Records.newRecord(Priority.class);
- priority.setPriority(stage.getPriority());
- ContainerAllocationEvent event =
- new ContainerAllocationEvent(ContainerAllocatorEventType.CONTAINER_REQ,
- stage.getId(), priority, resource, numRequest,
- stage.masterPlan.isLeaf(execBlock), 0.0f);
- stage.eventHandler.handle(event);
- }
-
private static void scheduleFragmentsForLeafQuery(Stage stage) throws IOException {
ExecutionBlock execBlock = stage.getBlock();
ScanNode[] scans = execBlock.getScanNodes();
@@ -1082,8 +1069,6 @@ public class Stage implements EventHandler<StageEvent> {
TableDesc table = stage.context.getTableDescMap().get(scan.getCanonicalName());
Collection<Fragment> fragments;
- TableMeta meta = table.getMeta();
-
Tablespace tablespace = TablespaceManager.get(scan.getTableDesc().getUri()).get();
// Depending on scanner node's type, it creates fragments. If scan is for
@@ -1157,69 +1142,6 @@ public class Stage implements EventHandler<StageEvent> {
return unit;
}
- private static class ContainerLaunchTransition
- implements SingleArcTransition<Stage, StageEvent> {
-
- @Override
- public void transition(Stage stage, StageEvent event) {
- if (!(event instanceof StageContainerAllocationEvent)) {
- throw new IllegalArgumentException("event should be a StageContainerAllocationEvent type.");
- }
- try {
- StageContainerAllocationEvent allocationEvent =
- (StageContainerAllocationEvent) event;
- for (TajoContainer container : allocationEvent.getAllocatedContainer()) {
- TajoContainerId cId = container.getId();
- if (stage.containers.containsKey(cId)) {
- stage.eventHandler.handle(new StageDiagnosticsUpdateEvent(stage.getId(),
- "Duplicated containers are allocated: " + cId.toString()));
- stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_INTERNAL_ERROR));
- }
- stage.containers.put(cId, container);
- }
- LOG.info("Stage (" + stage.getId() + ") has " + stage.containers.size() + " containers!");
- stage.eventHandler.handle(
- new LaunchTaskRunnersEvent(stage.getId(), allocationEvent.getAllocatedContainer(),
- stage.getContext().getQueryContext(),
- CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class))
- );
-
- stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_START));
- } catch (Throwable t) {
- stage.eventHandler.handle(new StageDiagnosticsUpdateEvent(stage.getId(),
- ExceptionUtils.getStackTrace(t)));
- stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_INTERNAL_ERROR));
- }
- }
- }
-
- /**
- * It is used in KILL_WAIT state against Contained Allocated event.
- * It just returns allocated containers to resource manager.
- */
- private static class AllocatedContainersCancelTransition implements SingleArcTransition<Stage, StageEvent> {
- @Override
- public void transition(Stage stage, StageEvent event) {
- if (!(event instanceof StageContainerAllocationEvent)) {
- throw new IllegalArgumentException("event should be a StageContainerAllocationEvent type.");
- }
- try {
- StageContainerAllocationEvent allocationEvent =
- (StageContainerAllocationEvent) event;
- stage.eventHandler.handle(
- new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_CLEANUP,
- stage.getId(), allocationEvent.getAllocatedContainer()));
- LOG.info(String.format("[%s] %d allocated containers are canceled",
- stage.getId().toString(),
- allocationEvent.getAllocatedContainer().size()));
- } catch (Throwable t) {
- stage.eventHandler.handle(new StageDiagnosticsUpdateEvent(stage.getId(),
- ExceptionUtils.getStackTrace(t)));
- stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_INTERNAL_ERROR));
- }
- }
- }
-
private static class TaskCompletedTransition implements SingleArcTransition<Stage, StageEvent> {
@Override
@@ -1236,6 +1158,7 @@ public class Stage implements EventHandler<StageEvent> {
stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_FAILED));
} else {
stage.completedTaskCount++;
+ stage.getTaskScheduler().releaseTaskAttempt(task.getLastAttempt());
if (taskEvent.getState() == TaskState.SUCCEEDED) {
stage.succeededObjectCount++;
@@ -1281,19 +1204,7 @@ public class Stage implements EventHandler<StageEvent> {
private void cleanup() {
stopScheduler();
- releaseContainers();
-
- if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) {
- List<ExecutionBlock> childs = getMasterPlan().getChilds(getId());
- List<TajoIdProtos.ExecutionBlockIdProto> ebIds = Lists.newArrayList();
-
- for (ExecutionBlock executionBlock : childs) {
- ebIds.add(executionBlock.getId().getProto());
- }
-
- getContext().getQueryMasterContext().getQueryMaster().cleanupExecutionBlock(ebIds);
- }
-
+ stopExecutionBlock();
this.finalStageHistory = makeStageHistory();
this.finalStageHistory.setTasks(makeTaskHistories());
}
@@ -1309,7 +1220,7 @@ public class Stage implements EventHandler<StageEvent> {
private void finalizeShuffleReport(StageShuffleReportEvent event, ShuffleType type) {
if(!checkIfNeedFinalizing(type)) return;
- TajoWorkerProtocol.ExecutionBlockReport report = event.getReport();
+ ExecutionBlockReport report = event.getReport();
if (!report.getReportSuccess()) {
stopFinalization();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
index 1da623e..a586e4b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
@@ -34,9 +34,8 @@ import org.apache.tajo.TajoProtos.TaskAttemptState;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TaskId;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.ipc.TajoWorkerProtocol.FailureIntermediateProto;
-import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto;
import org.apache.tajo.master.TaskState;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.plan.logical.*;
@@ -57,7 +56,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
+import static org.apache.tajo.ResourceProtos.*;
public class Task implements EventHandler<TaskEvent> {
/** Class Logger */
@@ -86,9 +85,8 @@ public class Task implements EventHandler<TaskEvent> {
private TaskAttemptId lastAttemptId;
private TaskAttemptId successfulAttempt;
- private String succeededHost;
- private int succeededHostPort;
- private int succeededPullServerPort;
+
+ private WorkerConnectionInfo succeededWorker;
private int failedAttempts;
private int finishedAttempts; // finish are total of success, failed and killed
@@ -174,6 +172,7 @@ public class Task implements EventHandler<TaskEvent> {
EnumSet.of(
TaskEventType.T_KILL,
TaskEventType.T_SCHEDULE,
+ TaskEventType.T_ATTEMPT_LAUNCHED,
TaskEventType.T_ATTEMPT_SUCCEEDED,
TaskEventType.T_ATTEMPT_FAILED))
@@ -252,7 +251,11 @@ public class Task implements EventHandler<TaskEvent> {
taskHistory.setState(lastAttempt.getState().toString());
taskHistory.setProgress(lastAttempt.getProgress());
}
- taskHistory.setHostAndPort(succeededHost + ":" + succeededHostPort);
+
+ if(getSucceededWorker() != null) {
+ taskHistory.setHostAndPort(succeededWorker.getHostAndPeerRpcPort());
+ }
+
taskHistory.setRetryCount(this.getRetryCount());
taskHistory.setLaunchTime(launchTime);
taskHistory.setFinishTime(finishTime);
@@ -357,8 +360,8 @@ public class Task implements EventHandler<TaskEvent> {
return dataLocations;
}
- public String getSucceededHost() {
- return succeededHost;
+ public WorkerConnectionInfo getSucceededWorker() {
+ return succeededWorker;
}
public void addFetches(String tableId, Collection<FetchImpl> fetches) {
@@ -608,9 +611,7 @@ public class Task implements EventHandler<TaskEvent> {
TaskAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
task.successfulAttempt = attemptEvent.getTaskAttemptId();
- task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
- task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
- task.succeededPullServerPort = attempt.getWorkerConnectionInfo().getPullServerPort();
+ task.succeededWorker = attempt.getWorkerConnectionInfo();
task.finishTask();
task.eventHandler.handle(new StageTaskEvent(event.getTaskId(), TaskState.SUCCEEDED));
@@ -624,11 +625,7 @@ public class Task implements EventHandler<TaskEvent> {
if (!(event instanceof TaskTAttemptEvent)) {
throw new IllegalArgumentException("event should be a TaskTAttemptEvent type.");
}
- TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
- TaskAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
task.launchTime = System.currentTimeMillis();
- task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
- task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java
index c1b9273..6c48d3b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java
@@ -23,18 +23,18 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
-import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoProtos.TaskAttemptState;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
+import org.apache.tajo.ResourceProtos.TaskCompletionReport;
+import org.apache.tajo.ResourceProtos.ShuffleFileOutput;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
import org.apache.tajo.querymaster.Task.IntermediateEntry;
import org.apache.tajo.querymaster.Task.PullHost;
-import org.apache.tajo.master.container.TajoContainerId;
import java.util.ArrayList;
import java.util.EnumSet;
@@ -43,8 +43,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
-
public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
private static final Log LOG = LogFactory.getLog(TaskAttempt.class);
@@ -55,7 +53,6 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
private final Task task;
final EventHandler eventHandler;
- private TajoContainerId containerId;
private WorkerConnectionInfo workerConnectionInfo;
private int expire;
@@ -109,6 +106,8 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
TaskAttemptEventType.TA_DONE, new SucceededTransition())
.addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_FAILED,
TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_UNASSIGNED,
+ TaskAttemptEventType.TA_ASSIGN_CANCEL, new CancelTransition())
// Transitions from TA_RUNNING state
.addTransition(TaskAttemptState.TA_RUNNING,
@@ -167,6 +166,10 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
TaskAttemptEventType.TA_ASSIGNED,
TaskAttemptEventType.TA_DONE),
new TaskKilledCompleteTransition())
+
+ // Transitions from TA_FAILED state
+ .addTransition(TaskAttemptState.TA_FAILED, TaskAttemptState.TA_FAILED,
+ TaskAttemptEventType.TA_KILL)
.installTopology();
private final StateMachine<TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
@@ -214,10 +217,6 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
return this.workerConnectionInfo;
}
- public void setContainerId(TajoContainerId containerId) {
- this.containerId = containerId;
- }
-
public synchronized void setExpireTime(int expire) {
this.expire = expire;
}
@@ -311,7 +310,6 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
throw new IllegalArgumentException("event should be a TaskAttemptAssignedEvent type.");
}
TaskAttemptAssignedEvent castEvent = (TaskAttemptAssignedEvent) event;
- taskAttempt.containerId = castEvent.getContainerId();
taskAttempt.workerConnectionInfo = castEvent.getWorkerConnectionInfo();
taskAttempt.eventHandler.handle(
new TaskTAttemptEvent(taskAttempt.getId(),
@@ -319,6 +317,17 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
}
}
+ private static class CancelTransition
+ implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent event) {
+
+ taskAttempt.workerConnectionInfo = null;
+ }
+ }
+
private static class TaskKilledCompleteTransition implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
@Override
@@ -396,7 +405,8 @@ public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
@Override
public void transition(TaskAttempt taskAttempt, TaskAttemptEvent event) {
- taskAttempt.eventHandler.handle(new LocalTaskEvent(taskAttempt.getId(), taskAttempt.containerId,
+ taskAttempt.eventHandler.handle(new LocalTaskEvent(taskAttempt.getId(),
+ taskAttempt.getWorkerConnectionInfo().getId(),
LocalTaskEventType.KILL));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/resource/DefaultResourceCalculator.java b/tajo-core/src/main/java/org/apache/tajo/resource/DefaultResourceCalculator.java
index 58b8a26..15ec845 100644
--- a/tajo-core/src/main/java/org/apache/tajo/resource/DefaultResourceCalculator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/resource/DefaultResourceCalculator.java
@@ -27,9 +27,7 @@ public class DefaultResourceCalculator extends ResourceCalculator {
@Override
public int computeAvailableContainers(NodeResource available, NodeResource required) {
- return Math.min(Math.min(
- available.getMemory() / required.getMemory(),
- available.getDisks() / required.getDisks()),
+ return Math.min(available.getMemory() / required.getMemory(),
available.getVirtualCores() / required.getVirtualCores());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/resource/NodeResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/resource/NodeResource.java b/tajo-core/src/main/java/org/apache/tajo/resource/NodeResource.java
index f51fc07..c7fe55b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/resource/NodeResource.java
+++ b/tajo-core/src/main/java/org/apache/tajo/resource/NodeResource.java
@@ -183,6 +183,6 @@ public class NodeResource implements ProtoObject<TajoProtos.NodeResourceProto>,
@Override
public String toString() {
- return "<memory:" + getMemory() + ", disks:" + getDisks() + ", vCores:" + getVirtualCores() + ">";
+ return "(Memory:" + getMemory() + ", Disks:" + getDisks() + ", vCores:" + getVirtualCores() + ")";
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/session/Session.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/session/Session.java
index ea7d29a..0d067dc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/session/Session.java
+++ b/tajo-core/src/main/java/org/apache/tajo/session/Session.java
@@ -32,7 +32,7 @@ import org.apache.tajo.common.ProtoObject;
import java.util.HashMap;
import java.util.Map;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.SessionProto;
+import static org.apache.tajo.ResourceProtos.SessionProto;
public class Session implements SessionConstants, ProtoObject<SessionProto>, Cloneable {
private static final Log LOG = LogFactory.getLog(Session.class);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
index 578b15a..7641320 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
@@ -19,7 +19,6 @@
package org.apache.tajo.util;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
@@ -31,8 +30,6 @@ import org.apache.tajo.querymaster.Task;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.util.history.StageHistory;
import org.apache.tajo.util.history.TaskHistory;
-import org.apache.tajo.worker.TaskRunner;
-import org.apache.tajo.worker.TaskRunnerHistory;
import java.text.DecimalFormat;
import java.util.*;
@@ -66,28 +63,6 @@ public class JSPUtil {
Collections.sort(tasks, new TaskHistoryComparator(sortField, "asc".equals(sortOrder)));
}
- public static void sortTaskRunner(List<TaskRunner> taskRunners) {
- Collections.sort(taskRunners, new Comparator<TaskRunner>() {
- @Override
- public int compare(TaskRunner taskRunner, TaskRunner taskRunner2) {
- return taskRunner.getId().compareTo(taskRunner2.getId());
- }
- });
- }
-
- public static void sortTaskRunnerHistory(List<TaskRunnerHistory> histories) {
- Collections.sort(histories, new Comparator<TaskRunnerHistory>() {
- @Override
- public int compare(TaskRunnerHistory h1, TaskRunnerHistory h2) {
- int value = h1.getExecutionBlockId().compareTo(h2.getExecutionBlockId());
- if(value == 0){
- return h1.getContainerId().compareTo(h2.getContainerId());
- }
- return value;
- }
- });
- }
-
public static String getElapsedTime(long startTime, long finishTime) {
if(startTime == 0) {
return "-";
@@ -221,8 +196,8 @@ public class JSPUtil {
if("id".equals(sortField)) {
return task.getId().compareTo(task2.getId());
} else if("host".equals(sortField)) {
- String host1 = task.getSucceededHost() == null ? "-" : task.getSucceededHost();
- String host2 = task2.getSucceededHost() == null ? "-" : task2.getSucceededHost();
+ String host1 = task.getSucceededWorker() == null ? "-" : task.getSucceededWorker().getHost();
+ String host2 = task2.getSucceededWorker() == null ? "-" : task2.getSucceededWorker().getHost();
return host1.compareTo(host2);
} else if("runTime".equals(sortField)) {
return compareLong(task.getRunningTime(), task2.getRunningTime());
@@ -235,8 +210,8 @@ public class JSPUtil {
if("id".equals(sortField)) {
return task2.getId().compareTo(task.getId());
} else if("host".equals(sortField)) {
- String host1 = task.getSucceededHost() == null ? "-" : task.getSucceededHost();
- String host2 = task2.getSucceededHost() == null ? "-" : task2.getSucceededHost();
+ String host1 = task.getSucceededWorker() == null ? "-" : task.getSucceededWorker().getHost();
+ String host2 = task2.getSucceededWorker() == null ? "-" : task2.getSucceededWorker().getHost();
return host2.compareTo(host1);
} else if("runTime".equals(sortField)) {
if(task2.getLaunchTime() == 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
index f4719b2..2acd5f6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskHistoryProto;
+import org.apache.tajo.ResourceProtos.TaskHistoryProto;
import org.apache.tajo.master.QueryInfo;
import org.apache.tajo.util.Bytes;
@@ -192,7 +192,11 @@ public class HistoryReader {
}
public List<TaskHistory> getTaskHistory(String queryId, String ebId) throws IOException {
- Path queryHistoryFile = getQueryHistoryFilePath(queryId, 0);
+ return getTaskHistory(queryId, ebId, 0);
+ }
+
+ public List<TaskHistory> getTaskHistory(String queryId, String ebId, long startTime) throws IOException {
+ Path queryHistoryFile = getQueryHistoryFilePath(queryId, startTime);
if (queryHistoryFile == null) {
return new ArrayList<TaskHistory>();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
index daced3e..c51b4e6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
@@ -53,6 +53,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class HistoryWriter extends AbstractService {
private static final Log LOG = LogFactory.getLog(HistoryWriter.class);
+ public static final String HISTORY_QUERY_REPLICATION = "tajo.history.query.replication";
+ public static final String HISTORY_TASK_REPLICATION = "tajo.history.task.replication";
+
public static final String QUERY_LIST = "query-list";
public static final String QUERY_DETAIL = "query-detail";
public static final String HISTORY_FILE_POSTFIX = ".hist";
@@ -87,13 +90,15 @@ public class HistoryWriter extends AbstractService {
if (!(conf instanceof TajoConf)) {
throw new IllegalArgumentException("conf should be a TajoConf type.");
}
- tajoConf = (TajoConf)conf;
+ tajoConf = (TajoConf) conf;
historyParentPath = tajoConf.getQueryHistoryDir(tajoConf);
taskHistoryParentPath = tajoConf.getTaskHistoryDir(tajoConf);
writerThread = new WriterThread();
historyCleaner = new HistoryCleaner(tajoConf, isMaster);
- queryReplication = (short) tajoConf.getIntVar(TajoConf.ConfVars.HISTORY_QUERY_REPLICATION);
- taskReplication = (short) tajoConf.getIntVar(TajoConf.ConfVars.HISTORY_TASK_REPLICATION);
+ queryReplication = (short) tajoConf.getInt(HISTORY_QUERY_REPLICATION,
+ FileSystem.get(tajoConf).getDefaultReplication(historyParentPath));
+ taskReplication = (short) tajoConf.getInt(HISTORY_TASK_REPLICATION,
+ FileSystem.get(tajoConf).getDefaultReplication(taskHistoryParentPath));
super.serviceInit(conf);
}
@@ -144,6 +149,7 @@ public class HistoryWriter extends AbstractService {
}
};
historyQueue.add(future);
+
synchronized (writerThread) {
writerThread.notifyAll();
}
@@ -151,7 +157,7 @@ public class HistoryWriter extends AbstractService {
}
/* synchronously flush to history file */
- public synchronized void appendAndSync(History history)
+ public void appendAndSync(History history)
throws TimeoutException, InterruptedException, IOException {
WriterFuture<WriterHolder> future = appendAndFlush(history);
@@ -227,7 +233,7 @@ public class HistoryWriter extends AbstractService {
}
try {
- if (!histories.isEmpty()) {
+ if (!stopped.get() && !histories.isEmpty()) {
writeHistory(histories);
} else {
continue;
@@ -239,25 +245,23 @@ public class HistoryWriter extends AbstractService {
//clean up history file
// closing previous writer
- synchronized (taskWriters) {
- Calendar cal = Calendar.getInstance();
- cal.add(Calendar.HOUR_OF_DAY, -2);
- String closeTargetTime = df.format(cal.getTime());
- List<String> closingTargets = new ArrayList<String>();
-
- for (String eachWriterTime : taskWriters.keySet()) {
- if (eachWriterTime.compareTo(closeTargetTime) <= 0) {
- closingTargets.add(eachWriterTime);
- }
+ Calendar cal = Calendar.getInstance();
+ cal.add(Calendar.HOUR_OF_DAY, -2);
+ String closeTargetTime = df.format(cal.getTime());
+ List<String> closingTargets = new ArrayList<String>();
+
+ for (String eachWriterTime : taskWriters.keySet()) {
+ if (eachWriterTime.compareTo(closeTargetTime) <= 0) {
+ closingTargets.add(eachWriterTime);
}
+ }
- for (String eachWriterTime : closingTargets) {
- WriterHolder writerHolder;
- writerHolder = taskWriters.remove(eachWriterTime);
- if (writerHolder != null) {
- LOG.info("Closing task history file: " + writerHolder.path);
- IOUtils.cleanup(LOG, writerHolder);
- }
+ for (String eachWriterTime : closingTargets) {
+ WriterHolder writerHolder;
+ writerHolder = taskWriters.remove(eachWriterTime);
+ if (writerHolder != null) {
+ LOG.info("Closing task history file: " + writerHolder.path);
+ IOUtils.cleanup(LOG, writerHolder);
}
}
}
@@ -267,24 +271,20 @@ public class HistoryWriter extends AbstractService {
private int drainHistory(Collection<WriterFuture<WriterHolder>> buffer, int numElements,
long timeoutMillis) throws InterruptedException {
- long deadline = System.currentTimeMillis() + timeoutMillis;
+ long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
int added = 0;
while (added < numElements) {
added += historyQueue.drainTo(buffer, numElements - added);
if (added < numElements) { // not enough elements immediately available; will have to wait
- if (deadline <= System.currentTimeMillis()) {
- break;
- } else {
- synchronized (writerThread) {
- writerThread.wait(deadline - System.currentTimeMillis());
- if (deadline > System.currentTimeMillis()) {
- added += historyQueue.drainTo(buffer, numElements - added);
- break;
- }
- }
+ WriterFuture<WriterHolder> e = historyQueue.poll(deadline - System.nanoTime(), TimeUnit.NANOSECONDS);
+ if (e == null) {
+ break; // we already waited enough, and there are no more elements in sight
}
+ buffer.add(e);
+ added++;
}
}
+
return added;
}
@@ -427,13 +427,11 @@ public class HistoryWriter extends AbstractService {
}
private void flushTaskHistories() {
- synchronized (taskWriters) {
- for (WriterHolder holder : taskWriters.values()) {
- try {
- holder.flush();
- } catch (IOException e) {
- LOG.warn(e, e);
- }
+ for (WriterHolder holder : taskWriters.values()) {
+ try {
+ holder.flush();
+ } catch (IOException e) {
+ LOG.warn(e, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
deleted file mode 100644
index 68c57f2..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.worker;
-
-import com.google.common.collect.Maps;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.tajo.master.ContainerProxy;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.container.TajoContainerId;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-
-public abstract class AbstractResourceAllocator extends CompositeService implements ResourceAllocator {
- /**
- * A key is worker id, and a value is a worker connection information.
- */
- protected ConcurrentMap<Integer, WorkerConnectionInfo> workerInfoMap = Maps.newConcurrentMap();
-
- public WorkerConnectionInfo getWorkerConnectionInfo(int workerId) {
- return workerInfoMap.get(workerId);
- }
-
- public void addWorkerConnectionInfo(WorkerConnectionInfo connectionInfo) {
- workerInfoMap.putIfAbsent(connectionInfo.getId(), connectionInfo);
- }
-
- private Map<TajoContainerId, ContainerProxy> containers = Maps.newConcurrentMap();
-
- public AbstractResourceAllocator() {
- super(AbstractResourceAllocator.class.getName());
- }
-
- public void addContainer(TajoContainerId cId, ContainerProxy container) {
- containers.put(cId, container);
- }
-
- public void removeContainer(TajoContainerId cId) {
- containers.remove(cId);
- }
-
- public boolean containsContainer(TajoContainerId cId) {
- return containers.containsKey(cId);
- }
-
- public ContainerProxy getContainer(TajoContainerId cId) {
- return containers.get(cId);
- }
-
- public Map<TajoContainerId, ContainerProxy> getContainers() {
- return containers;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
index cbd451d..57bedd2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
@@ -34,18 +34,15 @@ import org.apache.tajo.TaskId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.QueryMasterProtocol;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.rpc.*;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.storage.HashShuffleAppenderManager;
import org.apache.tajo.storage.StorageUtil;
-import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.worker.event.ExecutionBlockErrorEvent;
import java.io.IOException;
-import java.net.ConnectException;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -54,14 +51,14 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
+import static org.apache.tajo.ResourceProtos.*;
import static org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService.Interface;
public class ExecutionBlockContext {
/** class logger */
private static final Log LOG = LogFactory.getLog(ExecutionBlockContext.class);
- private TaskRunnerManager manager;
+ protected AtomicInteger runningTasksNum = new AtomicInteger();
protected AtomicInteger completedTasksNum = new AtomicInteger();
protected AtomicInteger succeededTasksNum = new AtomicInteger();
protected AtomicInteger killedTasksNum = new AtomicInteger();
@@ -79,10 +76,8 @@ public class ExecutionBlockContext {
private TajoQueryEngine queryEngine;
private RpcClientManager connManager;
- private InetSocketAddress qmMasterAddr;
- private NettyClientBase client;
+ private AsyncRpcClient queryMasterClient;
private QueryMasterProtocol.QueryMasterProtocolService.Interface stub;
- private WorkerConnectionInfo queryMaster;
private TajoConf systemConf;
// for the doAs block
private UserGroupInformation taskOwner;
@@ -96,17 +91,13 @@ public class ExecutionBlockContext {
// It keeps all of the query unit attempts while a TaskRunner is running.
private final ConcurrentMap<TaskAttemptId, Task> tasks = Maps.newConcurrentMap();
- @Deprecated
- private final ConcurrentMap<String, TaskRunnerHistory> histories = Maps.newConcurrentMap();
+ private final Map<TaskId, TaskHistory> taskHistories = Maps.newConcurrentMap();
- private final Map<TaskId, TaskHistory> taskHistories = Maps.newTreeMap();
-
- public ExecutionBlockContext(TajoWorker.WorkerContext workerContext,
- TaskRunnerManager manager, RunExecutionBlockRequestProto request) throws IOException {
- this.manager = manager;
+ public ExecutionBlockContext(TajoWorker.WorkerContext workerContext, ExecutionBlockContextResponse request,
+ AsyncRpcClient queryMasterClient)
+ throws IOException {
this.executionBlockId = new ExecutionBlockId(request.getExecutionBlockId());
this.connManager = RpcClientManager.getInstance();
- this.queryMaster = new WorkerConnectionInfo(request.getQueryMaster());
this.systemConf = workerContext.getConf();
this.reporter = new Reporter();
this.defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf);
@@ -120,6 +111,7 @@ public class ExecutionBlockContext {
this.resource = new ExecutionBlockSharedResource();
this.workerContext = workerContext;
this.shuffleType = request.getShuffleType();
+ this.queryMasterClient = queryMasterClient;
}
public void init() throws Throwable {
@@ -127,8 +119,6 @@ public class ExecutionBlockContext {
LOG.info("Tajo Root Dir: " + systemConf.getVar(TajoConf.ConfVars.ROOT_DIR));
LOG.info("Worker Local Dir: " + systemConf.getVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR));
- this.qmMasterAddr = NetUtils.createSocketAddr(queryMaster.getHost(), queryMaster.getQueryMasterPort());
- LOG.info("QueryMaster Address:" + qmMasterAddr);
UserGroupInformation.setConfiguration(systemConf);
// TODO - 'load credential' should be implemented
@@ -138,7 +128,7 @@ public class ExecutionBlockContext {
// initialize DFS and LocalFileSystems
this.taskOwner = taskOwner;
- this.stub = getRpcClient().getStub();
+ this.stub = queryMasterClient.getStub();
this.reporter.startReporter();
// resource intiailization
try{
@@ -157,12 +147,8 @@ public class ExecutionBlockContext {
return resource;
}
- private NettyClientBase getRpcClient()
- throws NoSuchMethodException, ConnectException, ClassNotFoundException {
- if (client != null) return client;
-
- client = connManager.newClient(qmMasterAddr, QueryMasterProtocol.class, true);
- return client;
+ private AsyncRpcClient getRpcClient() {
+ return queryMasterClient;
}
public Interface getStub() {
@@ -199,7 +185,7 @@ public class ExecutionBlockContext {
tasks.clear();
taskHistories.clear();
resource.release();
- RpcClientManager.cleanup(client);
+ RpcClientManager.cleanup(queryMasterClient);
}
public TajoConf getConf() {
@@ -256,21 +242,6 @@ public class ExecutionBlockContext {
return tasks.get(taskAttemptId);
}
- @Deprecated
- public void stopTaskRunner(String id){
- manager.stopTaskRunner(id);
- }
-
- @Deprecated
- public TaskRunner getTaskRunner(String taskRunnerId){
- return manager.getTaskRunner(taskRunnerId);
- }
-
- @Deprecated
- public void addTaskHistory(String taskRunnerId, TaskAttemptId quAttemptId, TaskHistory taskHistory) {
- histories.get(taskRunnerId).addTaskHistory(quAttemptId, taskHistory);
- }
-
public void addTaskHistory(TaskId taskId, TaskHistory taskHistory) {
taskHistories.put(taskId, taskHistory);
}
@@ -287,12 +258,15 @@ public class ExecutionBlockContext {
.setId(taskAttemptId.getProto())
.setErrorMessage(message);
- getStub().fatalError(null, builder.build(), NullCallback.get());
- }
-
- public TaskRunnerHistory createTaskRunnerHistory(TaskRunner runner){
- histories.putIfAbsent(runner.getId(), new TaskRunnerHistory(runner.getContainerId(), executionBlockId));
- return histories.get(runner.getId());
+ try {
+ //If QueryMaster does not responding, current execution block should be stop
+ CallFuture<PrimitiveProtos.NullProto> callFuture = new CallFuture<PrimitiveProtos.NullProto>();
+ getStub().fatalError(callFuture.getController(), builder.build(), callFuture);
+ callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ getWorkerContext().getTaskManager().getDispatcher().getEventHandler()
+ .handle(new ExecutionBlockErrorEvent(taskAttemptId.getTaskId().getExecutionBlockId(), e));
+ }
}
public TajoWorker.WorkerContext getWorkerContext(){
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
index 89c3404..07a9ba6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
@@ -22,10 +22,10 @@ import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.ResourceProtos.FetchProto;
import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.querymaster.Task;
import org.apache.tajo.querymaster.Repartitioner;
+import org.apache.tajo.querymaster.Task;
import org.apache.tajo.util.TUtil;
import java.net.URI;
@@ -37,7 +37,7 @@ import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
/**
* <code>FetchImpl</code> information to indicate the locations of intermediate data.
*/
-public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cloneable {
+public class FetchImpl implements ProtoObject<FetchProto>, Cloneable {
private Task.PullHost host; // The pull server host information
private ShuffleType type; // hash or range partition method.
private ExecutionBlockId executionBlockId; // The executionBlock id
@@ -57,7 +57,7 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
attemptIds = new ArrayList<Integer>();
}
- public FetchImpl(TajoWorkerProtocol.FetchProto proto) {
+ public FetchImpl(FetchProto proto) {
this(new Task.PullHost(proto.getHost(), proto.getPort()),
proto.getType(),
new ExecutionBlockId(proto.getExecutionBlockId()),
@@ -112,8 +112,8 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
}
@Override
- public TajoWorkerProtocol.FetchProto getProto() {
- TajoWorkerProtocol.FetchProto.Builder builder = TajoWorkerProtocol.FetchProto.newBuilder();
+ public FetchProto getProto() {
+ FetchProto.Builder builder = FetchProto.newBuilder();
builder.setHost(host.getHost());
builder.setPort(host.getPort());
[07/11] tajo git commit: TAJO-1397: Resource allocation should be
fine grained. (jinho)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEvent.java
new file mode 100644
index 0000000..856c572
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEvent.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler.event;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class SchedulerEvent extends AbstractEvent<SchedulerEventType> {
+ public SchedulerEvent(SchedulerEventType type) {
+ super(type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEventType.java
new file mode 100644
index 0000000..93fa032
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/event/SchedulerEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.scheduler.event;
+
+public enum SchedulerEventType {
+
+ // consumer: Scheduler
+ RESOURCE_RESERVE,
+ RESOURCE_UPDATE
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java b/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java
index 229a80a..200d689 100644
--- a/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java
@@ -22,8 +22,8 @@ import com.codahale.metrics.Gauge;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricSet;
import org.apache.tajo.master.TajoMaster;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerState;
+import org.apache.tajo.master.rm.NodeStatus;
+import org.apache.tajo.master.rm.NodeState;
import java.util.HashMap;
import java.util.Map;
@@ -40,31 +40,31 @@ public class WorkerResourceMetricsGaugeSet implements MetricSet {
metricsMap.put("totalWorkers", new Gauge<Integer>() {
@Override
public Integer getValue() {
- return tajoMasterContext.getResourceManager().getWorkers().size();
+ return tajoMasterContext.getResourceManager().getNodes().size();
}
});
metricsMap.put("liveWorkers", new Gauge<Integer>() {
@Override
public Integer getValue() {
- return getNumWorkers(WorkerState.RUNNING);
+ return getNumWorkers(NodeState.RUNNING);
}
});
metricsMap.put("deadWorkers", new Gauge<Integer>() {
@Override
public Integer getValue() {
- return getNumWorkers(WorkerState.LOST);
+ return getNumWorkers(NodeState.LOST);
}
});
return metricsMap;
}
- protected int getNumWorkers(WorkerState status) {
+ protected int getNumWorkers(NodeState status) {
int numWorkers = 0;
- for(Worker eachWorker: tajoMasterContext.getResourceManager().getWorkers().values()) {
- if(eachWorker.getState() == status) {
+ for(NodeStatus eachNodeStatus : tajoMasterContext.getResourceManager().getNodes().values()) {
+ if(eachNodeStatus.getState() == status) {
numWorkers++;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/AbstractTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/AbstractTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/AbstractTaskScheduler.java
index e45f274..8636eaa 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/AbstractTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/AbstractTaskScheduler.java
@@ -18,17 +18,21 @@
package org.apache.tajo.querymaster;
+import com.google.common.collect.Sets;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.tajo.master.event.TaskRequestEvent;
import org.apache.tajo.master.event.TaskSchedulerEvent;
+import java.util.Set;
+
public abstract class AbstractTaskScheduler extends AbstractService implements EventHandler<TaskSchedulerEvent> {
protected int hostLocalAssigned;
protected int rackLocalAssigned;
protected int totalAssigned;
+ protected int cancellation;
+ protected Set<String> leafTaskHosts = Sets.newHashSet();
/**
* Construct the service.
@@ -51,6 +55,15 @@ public abstract class AbstractTaskScheduler extends AbstractService implements E
return totalAssigned;
}
- public abstract void handleTaskRequestEvent(TaskRequestEvent event);
+ public int getCancellation() {
+ return cancellation;
+ }
+
+ public abstract void releaseTaskAttempt(TaskAttempt taskAttempt);
public abstract int remainingScheduledObjectNum();
+
+
+ public Set<String> getLeafTaskHosts(){
+ return leafTaskHosts;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java
index 939de60..32e4219 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java
@@ -20,56 +20,69 @@ package org.apache.tajo.querymaster;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.RackResolver;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
+import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.TaskRequest;
import org.apache.tajo.engine.query.TaskRequestImpl;
+import org.apache.tajo.ipc.QueryCoordinatorProtocol;
+import org.apache.tajo.ipc.QueryCoordinatorProtocol.QueryCoordinatorProtocolService;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.ContainerProxy;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
import org.apache.tajo.plan.serder.LogicalNodeSerializer;
-import org.apache.tajo.plan.serder.PlanProto;
+import org.apache.tajo.resource.NodeResources;
+import org.apache.tajo.rpc.*;
+import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.storage.DataLocation;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.FetchImpl;
+import java.net.InetSocketAddress;
import java.util.*;
import java.util.Map.Entry;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
+import static org.apache.tajo.ResourceProtos.*;
public class DefaultTaskScheduler extends AbstractTaskScheduler {
private static final Log LOG = LogFactory.getLog(DefaultTaskScheduler.class);
+ private static final String REQUEST_MAX_NUM = "tajo.qm.task-scheduler.request.max-num";
+
private final TaskSchedulerContext context;
private Stage stage;
+ private TajoConf tajoConf;
private Thread schedulingThread;
- private AtomicBoolean stopEventHandling = new AtomicBoolean(false);
+ private volatile boolean isStopped;
private ScheduledRequests scheduledRequests;
- private TaskRequests taskRequests;
+ private int minTaskMemory;
private int nextTaskId = 0;
private int scheduledObjectNum = 0;
+ private boolean isLeaf;
+ private int schedulerDelay;
+ private int maximumRequestContainer;
+
+ //candidate workers for locality of high priority
+ private Set<Integer> candidateWorkers = Sets.newHashSet();
public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) {
super(DefaultTaskScheduler.class.getName());
@@ -79,30 +92,47 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
@Override
public void init(Configuration conf) {
-
+ tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
scheduledRequests = new ScheduledRequests();
- taskRequests = new TaskRequests();
-
+ minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY);
+ schedulerDelay= tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_DELAY);
super.init(conf);
}
@Override
public void start() {
LOG.info("Start TaskScheduler");
+ maximumRequestContainer = tajoConf.getInt(REQUEST_MAX_NUM, stage.getContext().getWorkerMap().size() * 2);
+ isLeaf = stage.getMasterPlan().isLeaf(stage.getBlock());
+
+ if (isLeaf) {
+ candidateWorkers.addAll(getWorkerIds(getLeafTaskHosts()));
+ } else {
+ //find assigned hosts for Non-Leaf locality in children executionBlock
+ List<ExecutionBlock> executionBlockList = stage.getMasterPlan().getChilds(stage.getBlock());
+ for (ExecutionBlock executionBlock : executionBlockList) {
+ Stage childStage = stage.getContext().getStage(executionBlock.getId());
+ candidateWorkers.addAll(childStage.getAssignedWorkerMap().keySet());
+ }
+ }
this.schedulingThread = new Thread() {
public void run() {
- while(!stopEventHandling.get() && !Thread.currentThread().isInterrupted()) {
+ while (!isStopped && !Thread.currentThread().isInterrupted()) {
+
try {
- synchronized (schedulingThread){
- schedulingThread.wait(100);
- }
schedule();
} catch (InterruptedException e) {
- break;
+ if (isStopped) {
+ break;
+ } else {
+ LOG.fatal(e.getMessage(), e);
+ stage.abort(StageState.ERROR);
+ }
} catch (Throwable e) {
LOG.fatal(e.getMessage(), e);
+ stage.abort(StageState.ERROR);
break;
}
}
@@ -114,41 +144,17 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
super.start();
}
- private static final TaskAttemptId NULL_ATTEMPT_ID;
- public static final TajoWorkerProtocol.TaskRequestProto stopTaskRunnerReq;
- static {
- ExecutionBlockId nullStage = QueryIdFactory.newExecutionBlockId(QueryIdFactory.NULL_QUERY_ID, 0);
- NULL_ATTEMPT_ID = QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(nullStage, 0), 0);
-
- TajoWorkerProtocol.TaskRequestProto.Builder builder =
- TajoWorkerProtocol.TaskRequestProto.newBuilder();
- builder.setId(NULL_ATTEMPT_ID.getProto());
- builder.setShouldDie(true);
- builder.setOutputTable("");
- builder.setPlan(PlanProto.LogicalNodeTree.newBuilder());
- builder.setClusteredOutput(false);
- stopTaskRunnerReq = builder.build();
- }
-
@Override
public void stop() {
- if(stopEventHandling.getAndSet(true)){
- return;
- }
+ isStopped = true;
if (schedulingThread != null) {
synchronized (schedulingThread) {
- schedulingThread.notifyAll();
+ schedulingThread.interrupt();
}
}
-
- // Return all of request callbacks instantly.
- if(taskRequests != null){
- for (TaskRequestEvent req : taskRequests.taskRequestQueue) {
- req.getCallback().run(stopTaskRunnerReq);
- }
- }
-
+ candidateWorkers.clear();
+ scheduledRequests.clear();
LOG.info("Task Scheduler stopped");
super.stop();
}
@@ -156,34 +162,38 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
private Fragment[] fragmentsForNonLeafTask;
private Fragment[] broadcastFragmentsForNonLeafTask;
- LinkedList<TaskRequestEvent> taskRequestEvents = new LinkedList<TaskRequestEvent>();
- public void schedule() {
-
- if (taskRequests.size() > 0) {
- if (scheduledRequests.leafTaskNum() > 0) {
- LOG.debug("Try to schedule tasks with taskRequestEvents: " +
- taskRequests.size() + ", LeafTask Schedule Request: " +
- scheduledRequests.leafTaskNum());
- taskRequests.getTaskRequests(taskRequestEvents,
- scheduledRequests.leafTaskNum());
- LOG.debug("Get " + taskRequestEvents.size() + " taskRequestEvents ");
- if (taskRequestEvents.size() > 0) {
- scheduledRequests.assignToLeafTasks(taskRequestEvents);
- taskRequestEvents.clear();
+ public void schedule() throws Exception{
+ try {
+ if (remainingScheduledObjectNum() == 0) {
+ // all task is done, wait for stopping message
+ synchronized (schedulingThread) {
+ schedulingThread.wait(500);
}
- }
- }
+ } else {
+ LinkedList<TaskRequestEvent> taskRequests = createTaskRequest();
- if (taskRequests.size() > 0) {
- if (scheduledRequests.nonLeafTaskNum() > 0) {
- LOG.debug("Try to schedule tasks with taskRequestEvents: " +
- taskRequests.size() + ", NonLeafTask Schedule Request: " +
- scheduledRequests.nonLeafTaskNum());
- taskRequests.getTaskRequests(taskRequestEvents,
- scheduledRequests.nonLeafTaskNum());
- scheduledRequests.assignToNonLeafTasks(taskRequestEvents);
- taskRequestEvents.clear();
+ if (taskRequests.size() == 0) {
+ synchronized (schedulingThread) {
+ schedulingThread.wait(schedulerDelay);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get " + taskRequests.size() + " taskRequestEvents ");
+ }
+
+ if (isLeaf) {
+ if (scheduledRequests.leafTaskNum() > 0) {
+ scheduledRequests.assignToLeafTasks(taskRequests);
+ }
+ } else {
+ if (scheduledRequests.nonLeafTaskNum() > 0) {
+ scheduledRequests.assignToNonLeafTasks(taskRequests);
+ }
+ }
+ }
}
+ } catch (TimeoutException e) {
+ LOG.error(e.getMessage());
}
}
@@ -251,19 +261,55 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- @Override
- public void handleTaskRequestEvent(TaskRequestEvent event) {
-
- taskRequests.handle(event);
- int hosts = scheduledRequests.leafTaskHostMapping.size();
+ private Set<Integer> getWorkerIds(Collection<String> hosts){
+ Set<Integer> workerIds = Sets.newHashSet();
+ if(hosts.isEmpty()) return workerIds;
- // if available cluster resource are large then tasks, the scheduler thread are working immediately.
- if(remainingScheduledObjectNum() > 0 &&
- (remainingScheduledObjectNum() <= hosts || hosts <= taskRequests.size())){
- synchronized (schedulingThread){
- schedulingThread.notifyAll();
+ for (WorkerConnectionInfo worker : stage.getContext().getWorkerMap().values()) {
+ if(hosts.contains(worker.getHost())){
+ workerIds.add(worker.getId());
}
}
+ return workerIds;
+ }
+
+
+ protected LinkedList<TaskRequestEvent> createTaskRequest() throws Exception {
+ LinkedList<TaskRequestEvent> taskRequestEvents = new LinkedList<TaskRequestEvent>();
+
+ //If scheduled tasks is long-term task, cluster resource can be the worst load balance.
+ //This part is to throttle the maximum required container per request
+ int requestContainerNum = Math.min(remainingScheduledObjectNum(), maximumRequestContainer);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Try to schedule task resources: " + requestContainerNum);
+ }
+
+ ServiceTracker serviceTracker =
+ context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker();
+ NettyClientBase tmClient = RpcClientManager.getInstance().
+ getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true);
+ QueryCoordinatorProtocolService masterClientService = tmClient.getStub();
+
+ CallFuture<NodeResourceResponse> callBack = new CallFuture<NodeResourceResponse>();
+ NodeResourceRequest.Builder request = NodeResourceRequest.newBuilder();
+ request.setCapacity(NodeResources.createResource(minTaskMemory, isLeaf ? 1 : 0).getProto())
+ .setNumContainers(requestContainerNum)
+ .setPriority(stage.getPriority())
+ .setQueryId(context.getMasterContext().getQueryId().getProto())
+ .setType(isLeaf ? ResourceType.LEAF : ResourceType.INTERMEDIATE)
+ .setUserId(context.getMasterContext().getQueryContext().getUser())
+ .setRunningTasks(stage.getTotalScheduledObjectsCount() - stage.getCompletedTaskCount())
+ .addAllCandidateNodes(candidateWorkers)
+ .setQueue(context.getMasterContext().getQueryContext().get("queue", "default")); //TODO set queue
+
+ masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack);
+ NodeResourceResponse response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ for (AllocationResourceProto resource : response.getResourceList()) {
+ taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId()));
+ }
+
+ return taskRequestEvents;
}
@Override
@@ -271,43 +317,16 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
return scheduledObjectNum;
}
- private class TaskRequests implements EventHandler<TaskRequestEvent> {
- private final LinkedBlockingQueue<TaskRequestEvent> taskRequestQueue =
- new LinkedBlockingQueue<TaskRequestEvent>();
-
- @Override
- public void handle(TaskRequestEvent event) {
- if(LOG.isDebugEnabled()){
- LOG.debug("TaskRequest: " + event.getContainerId() + "," + event.getExecutionBlockId());
- }
+ public void releaseTaskAttempt(TaskAttempt taskAttempt) {
+ if (taskAttempt.isLeafTask() && taskAttempt.getWorkerConnectionInfo() != null) {
- if(stopEventHandling.get()) {
- event.getCallback().run(stopTaskRunnerReq);
- return;
+ HostVolumeMapping mapping =
+ scheduledRequests.leafTaskHostMapping.get(taskAttempt.getWorkerConnectionInfo().getHost());
+ if (mapping != null && mapping.lastAssignedVolumeId.containsKey(taskAttempt.getId())) {
+ mapping.decreaseConcurrency(mapping.lastAssignedVolumeId.remove(taskAttempt.getId()));
}
- int qSize = taskRequestQueue.size();
- if (qSize != 0 && qSize % 1000 == 0) {
- LOG.info("Size of event-queue in DefaultTaskScheduler is " + qSize);
- }
- int remCapacity = taskRequestQueue.remainingCapacity();
- if (remCapacity < 1000) {
- LOG.warn("Very low remaining capacity in the event-queue "
- + "of DefaultTaskScheduler: " + remCapacity);
- }
-
- taskRequestQueue.add(event);
- }
-
- public void getTaskRequests(final Collection<TaskRequestEvent> taskRequests,
- int num) {
- taskRequestQueue.drainTo(taskRequests, num);
- }
-
- public int size() {
- return taskRequestQueue.size();
}
}
-
/**
* One worker can have multiple running task runners. <code>HostVolumeMapping</code>
* describes various information for one worker, including :
@@ -342,8 +361,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
private Map<Integer, LinkedHashSet<TaskAttempt>> unassignedTaskForEachVolume =
Collections.synchronizedMap(new HashMap<Integer, LinkedHashSet<TaskAttempt>>());
/** A value is last assigned volume id for each task runner */
- private HashMap<TajoContainerId, Integer> lastAssignedVolumeId = new HashMap<TajoContainerId,
- Integer>();
+ private HashMap<TaskAttemptId, Integer> lastAssignedVolumeId = Maps.newHashMap();
/**
* A key is disk volume id, and a value is the load of this volume.
* This load is measured by counting how many number of tasks are running.
@@ -383,24 +401,18 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
* 2. unknown block or Non-splittable task in host
* 3. remote tasks. unassignedTaskForEachVolume is only contained local task. so it will be null
*/
- public synchronized TaskAttemptId getLocalTask(TajoContainerId containerId) {
- int volumeId;
+ public synchronized TaskAttemptId getLocalTask() {
+ int volumeId = getLowestVolumeId();
TaskAttemptId taskAttemptId = null;
- if (!lastAssignedVolumeId.containsKey(containerId)) {
- volumeId = getLowestVolumeId();
- increaseConcurrency(containerId, volumeId);
- } else {
- volumeId = lastAssignedVolumeId.get(containerId);
- }
-
+ increaseConcurrency(volumeId);
if (unassignedTaskForEachVolume.size() > 0) {
int retry = unassignedTaskForEachVolume.size();
do {
//clean and get a remaining local task
taskAttemptId = getAndRemove(volumeId);
if(!unassignedTaskForEachVolume.containsKey(volumeId)) {
- decreaseConcurrency(containerId);
+ decreaseConcurrency(volumeId);
if (volumeId > REMOTE) {
diskVolumeLoads.remove(volumeId);
}
@@ -409,7 +421,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
if (taskAttemptId == null) {
//reassign next volume
volumeId = getLowestVolumeId();
- increaseConcurrency(containerId, volumeId);
+ increaseConcurrency(volumeId);
retry--;
} else {
break;
@@ -418,6 +430,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
} else {
this.remainTasksNum.set(0);
}
+
+ lastAssignedVolumeId.put(taskAttemptId, volumeId);
return taskAttemptId;
}
@@ -490,11 +504,10 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
/**
* Increase the count of running tasks and disk loads for a certain task runner.
*
- * @param containerId The task runner identifier
* @param volumeId Volume identifier
* @return the volume load (i.e., how many running tasks use this volume)
*/
- private synchronized int increaseConcurrency(TajoContainerId containerId, int volumeId) {
+ private synchronized int increaseConcurrency(int volumeId) {
int concurrency = 1;
if (diskVolumeLoads.containsKey(volumeId)) {
@@ -512,16 +525,14 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
+ ", Remote Concurrency : " + concurrency);
}
diskVolumeLoads.put(volumeId, concurrency);
- lastAssignedVolumeId.put(containerId, volumeId);
return concurrency;
}
/**
* Decrease the count of running tasks of a certain task runner
*/
- private synchronized void decreaseConcurrency(TajoContainerId containerId){
- Integer volumeId = lastAssignedVolumeId.get(containerId);
- if(volumeId != null && diskVolumeLoads.containsKey(volumeId)){
+ private synchronized void decreaseConcurrency(int volumeId){
+ if(diskVolumeLoads.containsKey(volumeId)){
Integer concurrency = diskVolumeLoads.get(volumeId);
if(concurrency > 0){
diskVolumeLoads.put(volumeId, concurrency - 1);
@@ -531,7 +542,6 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
}
- lastAssignedVolumeId.remove(containerId);
}
/**
@@ -557,19 +567,6 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- public boolean isAssigned(TajoContainerId containerId){
- return lastAssignedVolumeId.containsKey(containerId);
- }
-
- public boolean isRemote(TajoContainerId containerId){
- Integer volumeId = lastAssignedVolumeId.get(containerId);
- if(volumeId == null || volumeId > REMOTE){
- return false;
- } else {
- return true;
- }
- }
-
public int getRemoteConcurrency(){
return getVolumeConcurrency(REMOTE);
}
@@ -585,7 +582,6 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
public String getHost() {
-
return host;
}
@@ -594,6 +590,25 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
+ public void cancel(TaskAttempt taskAttempt) {
+
+ if(taskAttempt.isLeafTask()) {
+ List<DataLocation> locations = taskAttempt.getTask().getDataLocations();
+
+ for (DataLocation location : locations) {
+ HostVolumeMapping volumeMapping = scheduledRequests.leafTaskHostMapping.get(location.getHost());
+ volumeMapping.addTaskAttempt(location.getVolumeId(), taskAttempt);
+ }
+
+ scheduledRequests.leafTasks.add(taskAttempt.getId());
+ } else {
+ scheduledRequests.nonLeafTasks.add(taskAttempt.getId());
+ }
+
+ context.getMasterContext().getEventHandler().handle(
+ new TaskAttemptEvent(taskAttempt.getId(), TaskAttemptEventType.TA_ASSIGN_CANCEL));
+ }
+
private class ScheduledRequests {
// two list leafTasks and nonLeafTasks keep all tasks to be scheduled. Even though some task is included in
// leafTaskHostMapping or leafTasksRackMapping, some task T will not be sent to a task runner
@@ -603,12 +618,20 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
private Map<String, HostVolumeMapping> leafTaskHostMapping = Maps.newConcurrentMap();
private final Map<String, HashSet<TaskAttemptId>> leafTasksRackMapping = Maps.newConcurrentMap();
- private synchronized void addLeafTask(TaskAttemptToSchedulerEvent event) {
+ protected void clear() {
+ leafTasks.clear();
+ nonLeafTasks.clear();
+ leafTaskHostMapping.clear();
+ leafTasksRackMapping.clear();
+ }
+
+ private void addLeafTask(TaskAttemptToSchedulerEvent event) {
TaskAttempt taskAttempt = event.getTaskAttempt();
List<DataLocation> locations = taskAttempt.getTask().getDataLocations();
for (DataLocation location : locations) {
String host = location.getHost();
+ leafTaskHosts.add(host);
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
if (hostVolumeMapping == null) {
@@ -650,14 +673,12 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
return nonLeafTasks.size();
}
- public Set<TaskAttemptId> assignedRequest = new HashSet<TaskAttemptId>();
-
- private TaskAttemptId allocateLocalTask(String host, TajoContainerId containerId){
+ private TaskAttemptId allocateLocalTask(String host){
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
if (hostVolumeMapping != null) { //tajo host is located in hadoop datanode
for (int i = 0; i < hostVolumeMapping.getRemainingLocalTaskSize(); i++) {
- TaskAttemptId attemptId = hostVolumeMapping.getLocalTask(containerId);
+ TaskAttemptId attemptId = hostVolumeMapping.getLocalTask();
if(attemptId == null) break;
//find remaining local task
@@ -736,8 +757,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
rackLocalAssigned++;
totalAssigned++;
- LOG.info(String.format("Assigned Local/Rack/Total: (%d/%d/%d), Locality: %.2f%%, Rack host: %s",
- hostLocalAssigned, rackLocalAssigned, totalAssigned,
+ LOG.info(String.format("Assigned Local/Rack/Cancel/Total: (%d/%d/%d/%d), Locality: %.2f%%, Rack host: %s",
+ hostLocalAssigned, rackLocalAssigned, cancellation, totalAssigned,
((double) hostLocalAssigned / (double) totalAssigned) * 100, host));
}
@@ -747,6 +768,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
Collections.shuffle(taskRequests);
LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<TaskRequestEvent>();
+ String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext().
+ getConnectionInfo().getHostAndQMPort();
TaskRequestEvent taskRequest;
while (leafTasks.size() > 0 && (!taskRequests.isEmpty() || !remoteTaskRequests.isEmpty())) {
@@ -757,58 +780,61 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
// checking if this container is still alive.
// If not, ignore the task request and stop the task runner
- ContainerProxy container = context.getMasterContext().getResourceAllocator()
- .getContainer(taskRequest.getContainerId());
- if(container == null) {
- taskRequest.getCallback().run(stopTaskRunnerReq);
- continue;
- }
+ WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());
+ if(connectionInfo == null) continue;
// getting the hostname of requested node
- WorkerConnectionInfo connectionInfo =
- context.getMasterContext().getResourceAllocator().getWorkerConnectionInfo(taskRequest.getWorkerId());
String host = connectionInfo.getHost();
// if there are no worker matched to the hostname a task request
- if(!leafTaskHostMapping.containsKey(host)){
+ if (!leafTaskHostMapping.containsKey(host) && !taskRequests.isEmpty()) {
String normalizedHost = NetUtils.normalizeHost(host);
- if(!leafTaskHostMapping.containsKey(normalizedHost) && !taskRequests.isEmpty()){
+ if (!leafTaskHostMapping.containsKey(normalizedHost)) {
// this case means one of either cases:
// * there are no blocks which reside in this node.
// * all blocks which reside in this node are consumed, and this task runner requests a remote task.
// In this case, we transfer the task request to the remote task request list, and skip the followings.
remoteTaskRequests.add(taskRequest);
continue;
+ } else {
+ host = normalizedHost;
}
}
- TajoContainerId containerId = taskRequest.getContainerId();
- LOG.debug("assignToLeafTasks: " + taskRequest.getExecutionBlockId() + "," +
- "containerId=" + containerId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("assignToLeafTasks: " + taskRequest.getExecutionBlockId() + "," +
+ "worker=" + connectionInfo.getHostAndPeerRpcPort());
+ }
//////////////////////////////////////////////////////////////////////
// disk or host-local allocation
//////////////////////////////////////////////////////////////////////
- TaskAttemptId attemptId = allocateLocalTask(host, containerId);
+ TaskAttemptId attemptId = allocateLocalTask(host);
if (attemptId == null) { // if a local task cannot be found
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
- if(hostVolumeMapping != null) {
- if(!hostVolumeMapping.isRemote(containerId)){
- // assign to remote volume
- hostVolumeMapping.decreaseConcurrency(containerId);
- hostVolumeMapping.increaseConcurrency(containerId, HostVolumeMapping.REMOTE);
- }
- // this part is remote concurrency management of a tail tasks
- int tailLimit = Math.max(remainingScheduledObjectNum() / (leafTaskHostMapping.size() * 2), 1);
+ if(!taskRequests.isEmpty()) { //if other requests remains, move to remote list for better locality
+ remoteTaskRequests.add(taskRequest);
+ candidateWorkers.remove(connectionInfo.getId());
+ continue;
- if(hostVolumeMapping.getRemoteConcurrency() > tailLimit){
- //release container
- hostVolumeMapping.decreaseConcurrency(containerId);
- taskRequest.getCallback().run(stopTaskRunnerReq);
- continue;
+ } else {
+ if(hostVolumeMapping != null) {
+ int nodes = context.getMasterContext().getWorkerMap().size();
+ //this part is to control the assignment of tail and remote task balancing per node
+ int tailLimit = 1;
+ if (remainingScheduledObjectNum() > 0) {
+ tailLimit = Math.max(remainingScheduledObjectNum() / nodes, 1);
+ }
+
+ if (hostVolumeMapping.getRemoteConcurrency() >= tailLimit) { //remote task throttling per node
+ continue;
+ } else {
+ // assign to remote volume
+ hostVolumeMapping.increaseConcurrency(HostVolumeMapping.REMOTE);
+ }
}
}
@@ -816,6 +842,9 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
// rack-local allocation
//////////////////////////////////////////////////////////////////////
attemptId = allocateRackTask(host);
+ if (attemptId != null && hostVolumeMapping != null) {
+ hostVolumeMapping.lastAssignedVolumeId.put(attemptId, HostVolumeMapping.REMOTE);
+ }
//////////////////////////////////////////////////////////////////////
// random node allocation
@@ -826,8 +855,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
leafTasks.remove(attemptId);
rackLocalAssigned++;
totalAssigned++;
- LOG.info(String.format("Assigned Local/Remote/Total: (%d/%d/%d), Locality: %.2f%%,",
- hostLocalAssigned, rackLocalAssigned, totalAssigned,
+ LOG.info(String.format("Assigned Local/Remote/Cancel/Total: (%d/%d/%d/%d), Locality: %.2f%%,",
+ hostLocalAssigned, rackLocalAssigned, cancellation, totalAssigned,
((double) hostLocalAssigned / (double) totalAssigned) * 100));
}
}
@@ -842,17 +871,51 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
false,
LogicalNodeSerializer.serialize(task.getLogicalPlan()),
context.getMasterContext().getQueryContext(),
- stage.getDataChannel(), stage.getBlock().getEnforcer());
+ stage.getDataChannel(), stage.getBlock().getEnforcer(),
+ queryMasterHostAndPort);
+
if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
taskAssign.setInterQuery();
}
- context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
- taskRequest.getContainerId(), connectionInfo));
- assignedRequest.add(attemptId);
+ //TODO send batch request
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
+ requestProto.addTaskRequest(TaskAllocationProto.newBuilder()
+ .setResource(taskRequest.getResponseProto().getResource())
+ .setTaskRequest(taskAssign.getProto()).build());
+
+ requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
+ context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
+ InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
+ if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
+
+ AsyncRpcClient tajoWorkerRpc = null;
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+ try {
+ tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
+ tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture);
+
+ BatchAllocationResponse responseProto =
+ callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ if (responseProto.getCancellationTaskCount() > 0) {
+ for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
+ cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
+ cancellation++;
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr);
+ }
+ continue;
+ }
+ } catch (Exception e) {
+ LOG.error(e);
+ }
scheduledObjectNum--;
- taskRequest.getCallback().run(taskAssign.getProto());
+
} else {
throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!");
}
@@ -874,6 +937,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
Collections.shuffle(taskRequests);
+ String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext().
+ getConnectionInfo().getHostAndQMPort();
TaskRequestEvent taskRequest;
while (!taskRequests.isEmpty()) {
@@ -891,6 +956,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
Task task;
task = stage.getTask(attemptId.getTaskId());
+
TaskRequest taskAssign = new TaskRequestImpl(
attemptId,
Lists.newArrayList(task.getAllFragments()),
@@ -899,7 +965,9 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
LogicalNodeSerializer.serialize(task.getLogicalPlan()),
context.getMasterContext().getQueryContext(),
stage.getDataChannel(),
- stage.getBlock().getEnforcer());
+ stage.getBlock().getEnforcer(),
+ queryMasterHostAndPort);
+
if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
taskAssign.setInterQuery();
}
@@ -912,13 +980,49 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- WorkerConnectionInfo connectionInfo = context.getMasterContext().getResourceAllocator().
- getWorkerConnectionInfo(taskRequest.getWorkerId());
- context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId,
- taskRequest.getContainerId(), connectionInfo));
- taskRequest.getCallback().run(taskAssign.getProto());
- totalAssigned++;
- scheduledObjectNum--;
+ WorkerConnectionInfo connectionInfo =
+ context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());
+
+ //TODO send batch request
+ BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
+ requestProto.addTaskRequest(TaskAllocationProto.newBuilder()
+ .setResource(taskRequest.getResponseProto().getResource())
+ .setTaskRequest(taskAssign.getProto()).build());
+
+ requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
+ context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
+
+ CallFuture<BatchAllocationResponse> callFuture = new CallFuture<BatchAllocationResponse>();
+
+ InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
+ if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
+
+ AsyncRpcClient tajoWorkerRpc;
+ try {
+ tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true);
+ TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
+ tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture);
+
+ BatchAllocationResponse
+ responseProto = callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+ if(responseProto.getCancellationTaskCount() > 0) {
+ for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
+ cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
+ cancellation++;
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr);
+ }
+ continue;
+ }
+
+ totalAssigned++;
+ scheduledObjectNum--;
+ } catch (Exception e) {
+ LOG.error(e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
index 9d5838d..6fc4ea9 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java
@@ -174,7 +174,16 @@ public class Query implements EventHandler<QueryEvent> {
QueryEventType.KILL,
QUERY_COMPLETED_TRANSITION)
- // Transitions from FAILED state
+ // Transitions from KILLED state
+ // ignore-able transitions
+ .addTransition(QueryState.QUERY_KILLED, QueryState.QUERY_KILLED,
+ EnumSet.of(QueryEventType.START, QueryEventType.QUERY_COMPLETED,
+ QueryEventType.KILL, QueryEventType.INTERNAL_ERROR))
+ .addTransition(QueryState.QUERY_KILLED, QueryState.QUERY_ERROR,
+ QueryEventType.INTERNAL_ERROR,
+ INTERNAL_ERROR_TRANSITION)
+
+ // Transitions from FAILED state
.addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_FAILED,
QueryEventType.DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
@@ -305,7 +314,6 @@ public class Query implements EventHandler<QueryEvent> {
queryHistory.setQueryId(getId().toString());
queryHistory.setQueryMaster(context.getQueryMasterContext().getWorkerContext().getWorkerName());
queryHistory.setHttpPort(context.getQueryMasterContext().getWorkerContext().getConnectionInfo().getHttpInfoPort());
- queryHistory.setLogicalPlan(plan.toString());
queryHistory.setLogicalPlan(plan.getLogicalPlan().toString());
queryHistory.setDistributedPlan(plan.toString());
@@ -704,8 +712,17 @@ public class Query implements EventHandler<QueryEvent> {
!executeNextBlock(query, castEvent.getExecutionBlockId())) {
return;
}
- // if a query is completed due to finished, kill, failure, or error
- query.eventHandler.handle(new QueryCompletedEvent(castEvent.getExecutionBlockId(), castEvent.getState()));
+
+ //wait for stages is completed
+ if (query.completedStagesCount >= query.stages.size()) {
+ // if a query is completed due to finished, kill, failure, or error
+ query.eventHandler.handle(new QueryCompletedEvent(castEvent.getExecutionBlockId(), castEvent.getState()));
+ }
+ LOG.info(String.format("Complete Stage[%s], State: %s, %d/%d. ",
+ castEvent.getExecutionBlockId().toString(),
+ castEvent.getState().toString(),
+ query.completedStagesCount,
+ query.stages.size()));
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
query.eventHandler.handle(new QueryEvent(event.getQueryId(), QueryEventType.INTERNAL_ERROR));
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
index 6c5bd22..e07b43f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
@@ -18,8 +18,8 @@
package org.apache.tajo.querymaster;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.commons.collections.map.LRUMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -29,19 +29,24 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.tajo.*;
+import org.apache.tajo.QueryId;
+
+import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.QueryCoordinatorProtocol;
-import org.apache.tajo.ipc.QueryCoordinatorProtocol.*;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ipc.QueryCoordinatorProtocol.QueryCoordinatorProtocolService;
+import org.apache.tajo.ResourceProtos.TajoHeartbeatRequest;
+import org.apache.tajo.ResourceProtos.TajoHeartbeatResponse;
+import org.apache.tajo.ResourceProtos.WorkerConnectionsResponse;
import org.apache.tajo.master.event.QueryStartEvent;
import org.apache.tajo.master.event.QueryStopEvent;
import org.apache.tajo.rpc.*;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.service.ServiceTracker;
-import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.history.HistoryReader;
import org.apache.tajo.util.history.QueryHistory;
import org.apache.tajo.worker.TajoWorker;
@@ -52,7 +57,6 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
public class QueryMaster extends CompositeService implements EventHandler {
private static final Log LOG = LogFactory.getLog(QueryMaster.class.getName());
@@ -69,11 +73,12 @@ public class QueryMaster extends CompositeService implements EventHandler {
private Map<QueryId, QueryMasterTask> queryMasterTasks = Maps.newConcurrentMap();
- private Map<QueryId, QueryMasterTask> finishedQueryMasterTasks = Maps.newConcurrentMap();
+ private final LRUMap
+ finishedQueryMasterTasksCache = new LRUMap(HistoryReader.DEFAULT_PAGE_SIZE);
private ClientSessionTimeoutCheckThread clientSessionTimeoutCheckThread;
- private AtomicBoolean queryMasterStop = new AtomicBoolean(false);
+ private volatile boolean isStopped;
private QueryMasterContext queryMasterContext;
@@ -89,44 +94,37 @@ public class QueryMaster extends CompositeService implements EventHandler {
private ExecutorService eventExecutor;
+ private ExecutorService singleEventExecutor;
+
public QueryMaster(TajoWorker.WorkerContext workerContext) {
super(QueryMaster.class.getName());
this.workerContext = workerContext;
}
- public void init(Configuration conf) {
- LOG.info("QueryMaster init");
- if (!(conf instanceof TajoConf)) {
- throw new IllegalArgumentException("conf should be a TajoConf type");
- }
- try {
- this.systemConf = (TajoConf)conf;
- this.manager = RpcClientManager.getInstance();
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
- querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT);
- queryMasterContext = new QueryMasterContext(systemConf);
+ this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+ this.manager = RpcClientManager.getInstance();
- clock = new SystemClock();
+ querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT);
+ queryMasterContext = new QueryMasterContext(systemConf);
- this.dispatcher = new AsyncDispatcher();
- addIfService(dispatcher);
+ clock = new SystemClock();
- globalPlanner = new GlobalPlanner(systemConf, workerContext);
+ this.dispatcher = new AsyncDispatcher();
+ addIfService(dispatcher);
- dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler());
- dispatcher.register(QueryStopEvent.EventType.class, new QueryStopEventHandler());
+ globalPlanner = new GlobalPlanner(systemConf, workerContext);
- } catch (Throwable t) {
- LOG.error(t.getMessage(), t);
- throw new RuntimeException(t);
- }
- super.init(conf);
+ dispatcher.register(QueryStartEvent.EventType.class, new QueryStartEventHandler());
+ dispatcher.register(QueryStopEvent.EventType.class, new QueryStopEventHandler());
+ super.serviceInit(conf);
+ LOG.info("QueryMaster inited");
}
@Override
- public void start() {
- LOG.info("QueryMaster start");
-
+ public void serviceStart() throws Exception {
queryHeartbeatThread = new QueryHeartbeatThread();
queryHeartbeatThread.start();
@@ -136,15 +134,15 @@ public class QueryMaster extends CompositeService implements EventHandler {
finishedQueryMasterTaskCleanThread = new FinishedQueryMasterTaskCleanThread();
finishedQueryMasterTaskCleanThread.start();
- eventExecutor = Executors.newSingleThreadExecutor();
- super.start();
+ eventExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+ singleEventExecutor = Executors.newSingleThreadExecutor();
+ super.serviceStart();
+ LOG.info("QueryMaster started");
}
@Override
- public void stop() {
- if(queryMasterStop.getAndSet(true)){
- return;
- }
+ public void serviceStop() throws Exception {
+ isStopped = true;
if(queryHeartbeatThread != null) {
queryHeartbeatThread.interrupt();
@@ -162,60 +160,15 @@ public class QueryMaster extends CompositeService implements EventHandler {
eventExecutor.shutdown();
}
- super.stop();
-
- LOG.info("QueryMaster stopped");
- }
-
- protected void cleanupExecutionBlock(List<TajoIdProtos.ExecutionBlockIdProto> executionBlockIds) {
- StringBuilder cleanupMessage = new StringBuilder();
- String prefix = "";
- for (TajoIdProtos.ExecutionBlockIdProto eachEbId: executionBlockIds) {
- cleanupMessage.append(prefix).append(new ExecutionBlockId(eachEbId).toString());
- prefix = ",";
+ if(singleEventExecutor != null){
+ singleEventExecutor.shutdown();
}
- LOG.info("cleanup executionBlocks: " + cleanupMessage);
- NettyClientBase rpc = null;
- List<WorkerResourceProto> workers = getAllWorker();
- TajoWorkerProtocol.ExecutionBlockListProto.Builder builder = TajoWorkerProtocol.ExecutionBlockListProto.newBuilder();
- builder.addAllExecutionBlockId(Lists.newArrayList(executionBlockIds));
- TajoWorkerProtocol.ExecutionBlockListProto executionBlockListProto = builder.build();
- for (WorkerResourceProto worker : workers) {
- try {
- TajoProtos.WorkerConnectionInfoProto connectionInfo = worker.getConnectionInfo();
- rpc = manager.getClient(NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()),
- TajoWorkerProtocol.class, true);
- TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub();
-
- tajoWorkerProtocolService.cleanupExecutionBlocks(null, executionBlockListProto, NullCallback.get());
- } catch (Exception e) {
- LOG.warn("Ignoring exception. " + e.getMessage(), e);
- continue;
- }
- }
- }
-
- private void cleanup(QueryId queryId) {
- LOG.info("cleanup query resources : " + queryId);
- NettyClientBase rpc = null;
- List<WorkerResourceProto> workers = getAllWorker();
-
- for (WorkerResourceProto worker : workers) {
- try {
- TajoProtos.WorkerConnectionInfoProto connectionInfo = worker.getConnectionInfo();
- rpc = manager.getClient(NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()),
- TajoWorkerProtocol.class, true);
- TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub();
-
- tajoWorkerProtocolService.cleanup(null, queryId.getProto(), NullCallback.get());
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
+ super.serviceStop();
+ LOG.info("QueryMaster stopped");
}
- public List<WorkerResourceProto> getAllWorker() {
+ public List<TajoProtos.WorkerConnectionInfoProto> getAllWorker() {
NettyClientBase rpc = null;
try {
@@ -228,16 +181,17 @@ public class QueryMaster extends CompositeService implements EventHandler {
rpc = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true);
QueryCoordinatorProtocolService masterService = rpc.getStub();
- CallFuture<WorkerResourcesRequest> callBack = new CallFuture<WorkerResourcesRequest>();
- masterService.getAllWorkerResource(callBack.getController(),
+ CallFuture<WorkerConnectionsResponse> callBack = new CallFuture<WorkerConnectionsResponse>();
+ masterService.getAllWorkers(callBack.getController(),
PrimitiveProtos.NullProto.getDefaultInstance(), callBack);
- WorkerResourcesRequest workerResourcesRequest = callBack.get(2, TimeUnit.SECONDS);
- return workerResourcesRequest.getWorkerResourcesList();
+ WorkerConnectionsResponse connectionsProto =
+ callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ return connectionsProto.getWorkerList();
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
- return new ArrayList<WorkerResourceProto>();
+ return new ArrayList<TajoProtos.WorkerConnectionInfoProto>();
}
@Override
@@ -253,13 +207,14 @@ public class QueryMaster extends CompositeService implements EventHandler {
return queryMasterTasks.get(queryId);
}
+ @Deprecated
public QueryMasterTask getQueryMasterTask(QueryId queryId, boolean includeFinished) {
QueryMasterTask queryMasterTask = queryMasterTasks.get(queryId);
if(queryMasterTask != null) {
return queryMasterTask;
} else {
if(includeFinished) {
- return finishedQueryMasterTasks.get(queryId);
+ return (QueryMasterTask) finishedQueryMasterTasksCache.get(queryId);
} else {
return null;
}
@@ -274,8 +229,9 @@ public class QueryMaster extends CompositeService implements EventHandler {
return queryMasterTasks.values();
}
+ @Deprecated
public Collection<QueryMasterTask> getFinishedQueryMasterTasks() {
- return finishedQueryMasterTasks.values();
+ return finishedQueryMasterTasksCache.values();
}
public class QueryMasterContext {
@@ -293,6 +249,10 @@ public class QueryMaster extends CompositeService implements EventHandler {
return eventExecutor;
}
+ public ExecutorService getSingleEventExecutor(){
+ return singleEventExecutor;
+ }
+
public AsyncDispatcher getDispatcher() {
return dispatcher;
}
@@ -324,12 +284,12 @@ public class QueryMaster extends CompositeService implements EventHandler {
return;
}
- finishedQueryMasterTasks.put(queryId, queryMasterTask);
+ finishedQueryMasterTasksCache.put(queryId, queryMasterTask);
- TajoHeartbeat queryHeartbeat = buildTajoHeartBeat(queryMasterTask);
+ TajoHeartbeatRequest queryHeartbeat = buildTajoHeartBeat(queryMasterTask);
CallFuture<TajoHeartbeatResponse> future = new CallFuture<TajoHeartbeatResponse>();
- NettyClientBase tmClient = null;
+ NettyClientBase tmClient;
try {
tmClient = manager.getClient(workerContext.getServiceTracker().getUmbilicalAddress(),
QueryCoordinatorProtocol.class, true);
@@ -346,9 +306,6 @@ public class QueryMaster extends CompositeService implements EventHandler {
try {
queryMasterTask.stop();
- if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) {
- cleanup(queryId);
- }
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
@@ -367,8 +324,8 @@ public class QueryMaster extends CompositeService implements EventHandler {
}
}
- private TajoHeartbeat buildTajoHeartBeat(QueryMasterTask queryMasterTask) {
- TajoHeartbeat.Builder builder = TajoHeartbeat.newBuilder();
+ private TajoHeartbeatRequest buildTajoHeartBeat(QueryMasterTask queryMasterTask) {
+ TajoHeartbeatRequest.Builder builder = TajoHeartbeatRequest.newBuilder();
builder.setConnectionInfo(workerContext.getConnectionInfo().getProto());
builder.setQueryId(queryMasterTask.getQueryId().getProto());
@@ -387,7 +344,7 @@ public class QueryMaster extends CompositeService implements EventHandler {
public void handle(QueryStartEvent event) {
LOG.info("Start QueryStartEventHandler:" + event.getQueryId());
QueryMasterTask queryMasterTask = new QueryMasterTask(queryMasterContext,
- event.getQueryId(), event.getSession(), event.getQueryContext(), event.getJsonExpr());
+ event.getQueryId(), event.getSession(), event.getQueryContext(), event.getJsonExpr(), event.getAllocation());
synchronized(queryMasterTasks) {
queryMasterTasks.put(event.getQueryId(), queryMasterTask);
@@ -402,7 +359,6 @@ public class QueryMaster extends CompositeService implements EventHandler {
if (queryMasterTask.isInitError()) {
queryMasterContext.stopQuery(queryMasterTask.getQueryId());
- return;
}
}
}
@@ -422,31 +378,29 @@ public class QueryMaster extends CompositeService implements EventHandler {
@Override
public void run() {
LOG.info("Start QueryMaster heartbeat thread");
- while(!queryMasterStop.get()) {
+ while(!isStopped) {
List<QueryMasterTask> tempTasks = new ArrayList<QueryMasterTask>();
- synchronized(queryMasterTasks) {
- tempTasks.addAll(queryMasterTasks.values());
- }
- synchronized(queryMasterTasks) {
- for(QueryMasterTask eachTask: tempTasks) {
- NettyClientBase tmClient;
- try {
+ tempTasks.addAll(queryMasterTasks.values());
+
+ for(QueryMasterTask eachTask: tempTasks) {
+ NettyClientBase tmClient;
+ try {
- ServiceTracker serviceTracker = queryMasterContext.getWorkerContext().getServiceTracker();
- tmClient = manager.getClient(serviceTracker.getUmbilicalAddress(),
- QueryCoordinatorProtocol.class, true);
- QueryCoordinatorProtocolService masterClientService = tmClient.getStub();
+ ServiceTracker serviceTracker = queryMasterContext.getWorkerContext().getServiceTracker();
+ tmClient = manager.getClient(serviceTracker.getUmbilicalAddress(),
+ QueryCoordinatorProtocol.class, true);
+ QueryCoordinatorProtocolService masterClientService = tmClient.getStub();
- TajoHeartbeat queryHeartbeat = buildTajoHeartBeat(eachTask);
- masterClientService.heartbeat(null, queryHeartbeat, NullCallback.get());
- } catch (Throwable t) {
- t.printStackTrace();
- }
+ TajoHeartbeatRequest queryHeartbeat = buildTajoHeartBeat(eachTask);
+ masterClientService.heartbeat(null, queryHeartbeat, NullCallback.get());
+ } catch (Throwable t) {
+ t.printStackTrace();
}
}
- synchronized(queryMasterStop) {
+
+ synchronized(this) {
try {
- queryMasterStop.wait(2000);
+ this.wait(2000);
} catch (InterruptedException e) {
break;
}
@@ -459,16 +413,16 @@ public class QueryMaster extends CompositeService implements EventHandler {
class ClientSessionTimeoutCheckThread extends Thread {
public void run() {
LOG.info("ClientSessionTimeoutCheckThread started");
- while(!queryMasterStop.get()) {
+ while(!isStopped) {
try {
- Thread.sleep(1000);
+ synchronized (this) {
+ this.wait(1000);
+ }
} catch (InterruptedException e) {
break;
}
List<QueryMasterTask> tempTasks = new ArrayList<QueryMasterTask>();
- synchronized(queryMasterTasks) {
- tempTasks.addAll(queryMasterTasks.values());
- }
+ tempTasks.addAll(queryMasterTasks.values());
for(QueryMasterTask eachTask: tempTasks) {
if(!eachTask.isStopped()) {
@@ -490,11 +444,13 @@ public class QueryMaster extends CompositeService implements EventHandler {
class FinishedQueryMasterTaskCleanThread extends Thread {
public void run() {
- int expireIntervalTime = systemConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_HISTORY_EXPIRE_PERIOD);
+ int expireIntervalTime = systemConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_CACHE_EXPIRE_PERIOD);
LOG.info("FinishedQueryMasterTaskCleanThread started: expire interval minutes = " + expireIntervalTime);
- while(!queryMasterStop.get()) {
+ while(!isStopped) {
try {
- Thread.sleep(60 * 1000); // minimum interval minutes
+ synchronized (this) {
+ this.wait(60 * 1000); // minimum interval minutes
+ }
} catch (InterruptedException e) {
break;
}
@@ -508,26 +464,25 @@ public class QueryMaster extends CompositeService implements EventHandler {
}
private void cleanExpiredFinishedQueryMasterTask(long expireTime) {
- synchronized(finishedQueryMasterTasks) {
- List<QueryId> expiredQueryIds = new ArrayList<QueryId>();
- for(Map.Entry<QueryId, QueryMasterTask> entry: finishedQueryMasterTasks.entrySet()) {
-
+ List<QueryId> expiredQueryIds = new ArrayList<QueryId>();
+ for(Object key: new ArrayList<Object>(finishedQueryMasterTasksCache.keySet())) {
+ QueryId queryId = (QueryId) key;
/* If a query are abnormal termination, the finished time will be zero. */
- long finishedTime = entry.getValue().getStartTime();
- Query query = entry.getValue().getQuery();
- if (query != null && query.getFinishTime() > 0) {
- finishedTime = query.getFinishTime();
- }
-
- if(finishedTime < expireTime) {
- expiredQueryIds.add(entry.getKey());
- }
+ QueryMasterTask queryMasterTask = (QueryMasterTask) finishedQueryMasterTasksCache.get(queryId);
+ long finishedTime = queryMasterTask.getStartTime();
+ Query query = queryMasterTask.getQuery();
+ if (query != null && query.getFinishTime() > 0) {
+ finishedTime = query.getFinishTime();
}
- for(QueryId eachId: expiredQueryIds) {
- finishedQueryMasterTasks.remove(eachId);
+ if(finishedTime < expireTime) {
+ expiredQueryIds.add(queryId);
}
}
+
+ for(QueryId eachId: expiredQueryIds) {
+ finishedQueryMasterTasksCache.remove(eachId);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a02873d/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java
index 59933a7..62216aa 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java
@@ -18,7 +18,6 @@
package org.apache.tajo.querymaster;
-import com.google.common.base.Preconditions;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import org.apache.commons.logging.Log;
@@ -27,19 +26,26 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.tajo.*;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.QueryMasterProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.event.*;
-import org.apache.tajo.session.Session;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.plan.serder.PlanProto;
+import org.apache.tajo.resource.NodeResource;
import org.apache.tajo.rpc.AsyncRpcServer;
+import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
+import org.apache.tajo.session.Session;
import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TajoWorker;
+import org.apache.tajo.worker.event.QMResourceAllocateEvent;
import java.net.InetSocketAddress;
+import static org.apache.tajo.ResourceProtos.*;
+
public class QueryMasterManagerService extends CompositeService
implements QueryMasterProtocol.QueryMasterProtocolService.Interface {
private static final Log LOG = LogFactory.getLog(QueryMasterManagerService.class.getName());
@@ -64,51 +70,41 @@ public class QueryMasterManagerService extends CompositeService
}
@Override
- public void init(Configuration conf) {
- Preconditions.checkArgument(conf instanceof TajoConf);
- TajoConf tajoConf = (TajoConf) conf;
- try {
- // Setup RPC server
- InetSocketAddress initIsa =
- new InetSocketAddress("0.0.0.0", port);
- if (initIsa.getAddress() == null) {
- throw new IllegalArgumentException("Failed resolve of " + initIsa);
- }
-
- int workerNum = tajoConf.getIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM);
- this.rpcServer = new AsyncRpcServer(QueryMasterProtocol.class, this, initIsa, workerNum);
- this.rpcServer.start();
+ public void serviceInit(Configuration conf) throws Exception {
+
+ TajoConf tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
+ // Setup RPC server
+ InetSocketAddress initIsa =
+ new InetSocketAddress("0.0.0.0", port);
+ if (initIsa.getAddress() == null) {
+ throw new IllegalArgumentException("Failed resolve of " + initIsa);
+ }
- this.bindAddr = NetUtils.getConnectAddress(rpcServer.getListenAddress());
- this.addr = bindAddr.getHostName() + ":" + bindAddr.getPort();
+ int workerNum = tajoConf.getIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM);
+ this.rpcServer = new AsyncRpcServer(QueryMasterProtocol.class, this, initIsa, workerNum);
+ this.rpcServer.start();
- this.port = bindAddr.getPort();
+ this.bindAddr = NetUtils.getConnectAddress(rpcServer.getListenAddress());
+ this.addr = bindAddr.getHostName() + ":" + bindAddr.getPort();
- queryMaster = new QueryMaster(workerContext);
- addService(queryMaster);
+ this.port = bindAddr.getPort();
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
+ queryMaster = new QueryMaster(workerContext);
+ addService(queryMaster);
// Get the master address
LOG.info("QueryMasterManagerService is bind to " + addr);
- ((TajoConf)conf).setVar(TajoConf.ConfVars.WORKER_QM_RPC_ADDRESS, addr);
+ tajoConf.setVar(TajoConf.ConfVars.WORKER_QM_RPC_ADDRESS, addr);
- super.init(conf);
+ super.serviceInit(conf);
}
@Override
- public void start() {
- super.start();
- }
-
- @Override
- public void stop() {
+ public void serviceStop() throws Exception {
if(rpcServer != null) {
rpcServer.shutdown();
}
LOG.info("QueryMasterManagerService stopped");
- super.stop();
+ super.serviceStop();
}
public InetSocketAddress getBindAddr() {
@@ -116,52 +112,31 @@ public class QueryMasterManagerService extends CompositeService
}
@Override
- public void getTask(RpcController controller, TajoWorkerProtocol.GetTaskRequestProto request,
- RpcCallback<TajoWorkerProtocol.TaskRequestProto> done) {
- try {
- ExecutionBlockId ebId = new ExecutionBlockId(request.getExecutionBlockId());
- QueryMasterTask queryMasterTask = workerContext.getQueryMaster().getQueryMasterTask(ebId.getQueryId());
-
- if(queryMasterTask == null || queryMasterTask.isStopped()) {
- done.run(DefaultTaskScheduler.stopTaskRunnerReq);
- } else {
- TajoContainerId cid =
- queryMasterTask.getQueryTaskContext().getResourceAllocator().makeContainerId(request.getContainerId());
- LOG.debug("getTask:" + cid + ", ebId:" + ebId);
- queryMasterTask.handleTaskRequestEvent(new TaskRequestEvent(request.getWorkerId(), cid, ebId, done));
- }
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- controller.setFailed(e.getMessage());
- }
- }
-
- @Override
- public void statusUpdate(RpcController controller, TajoWorkerProtocol.TaskStatusProto request,
+ public void statusUpdate(RpcController controller, TaskStatusProto request,
RpcCallback<PrimitiveProtos.NullProto> done) {
QueryId queryId = new QueryId(request.getId().getTaskId().getExecutionBlockId().getQueryId());
TaskAttemptId attemptId = new TaskAttemptId(request.getId());
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(queryId);
- if (queryMasterTask == null) {
- queryMasterTask = queryMaster.getQueryMasterTask(queryId, true);
- }
- Stage sq = queryMasterTask.getQuery().getStage(attemptId.getTaskId().getExecutionBlockId());
- Task task = sq.getTask(attemptId.getTaskId());
- TaskAttempt attempt = task.getAttempt(attemptId.getId());
- if(LOG.isDebugEnabled()){
- LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name()));
- }
+ if (queryMasterTask != null) {
+ Stage sq = queryMasterTask.getQuery().getStage(attemptId.getTaskId().getExecutionBlockId());
+ Task task = sq.getTask(attemptId.getTaskId());
+ TaskAttempt attempt = task.getAttempt(attemptId.getId());
- if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) {
- LOG.warn(attemptId + " Killed");
- attempt.handle(
- new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED));
- } else {
- queryMasterTask.getEventHandler().handle(
- new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request));
- }
+ if(LOG.isDebugEnabled()){
+ LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name()));
+ }
+
+ if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) {
+ LOG.warn(attemptId + " Killed");
+ attempt.handle(
+ new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED));
+ } else {
+ queryMasterTask.getEventHandler().handle(
+ new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request));
+ }
+ }
done.run(TajoWorker.NULL_PROTO);
}
@@ -173,7 +148,7 @@ public class QueryMasterManagerService extends CompositeService
}
@Override
- public void fatalError(RpcController controller, TajoWorkerProtocol.TaskFatalErrorReport report,
+ public void fatalError(RpcController controller, TaskFatalErrorReport report,
RpcCallback<PrimitiveProtos.NullProto> done) {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
new QueryId(report.getId().getTaskId().getExecutionBlockId().getQueryId()));
@@ -186,7 +161,7 @@ public class QueryMasterManagerService extends CompositeService
}
@Override
- public void done(RpcController controller, TajoWorkerProtocol.TaskCompletionReport report,
+ public void done(RpcController controller, TaskCompletionReport report,
RpcCallback<PrimitiveProtos.NullProto> done) {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
new QueryId(report.getId().getTaskId().getExecutionBlockId().getQueryId()));
@@ -198,7 +173,7 @@ public class QueryMasterManagerService extends CompositeService
@Override
public void doneExecutionBlock(
- RpcController controller, TajoWorkerProtocol.ExecutionBlockReport request,
+ RpcController controller, ExecutionBlockReport request,
RpcCallback<PrimitiveProtos.NullProto> done) {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(new QueryId(request.getEbId().getQueryId()));
if (queryMasterTask != null) {
@@ -209,6 +184,38 @@ public class QueryMasterManagerService extends CompositeService
}
@Override
+ public void getExecutionBlockContext(RpcController controller,
+ ExecutionBlockContextRequest request,
+ RpcCallback<ExecutionBlockContextResponse> done) {
+
+ QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
+ new QueryId(request.getExecutionBlockId().getQueryId()));
+ if (queryMasterTask != null) {
+
+ Stage stage = queryMasterTask.getQuery().getStage(new ExecutionBlockId(request.getExecutionBlockId()));
+
+ // first request with starting ExecutionBlock
+ PlanProto.ShuffleType shuffleType = stage.getDataChannel().getShuffleType();
+
+ ExecutionBlockContextResponse.Builder ebRequestProto = ExecutionBlockContextResponse.newBuilder();
+ ebRequestProto.setExecutionBlockId(request.getExecutionBlockId())
+ .setQueryContext(stage.getContext().getQueryContext().getProto())
+ .setQueryOutputPath(stage.getContext().getStagingDir().toString())
+ .setPlanJson(CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class))
+ .setShuffleType(shuffleType);
+
+ //Set assigned worker to stage
+ if (!stage.getAssignedWorkerMap().containsKey(request.getWorker().getId())) {
+ stage.getAssignedWorkerMap().put(request.getWorker().getId(),
+ NetUtils.createSocketAddr(request.getWorker().getHost(), request.getWorker().getPeerRpcPort()));
+ }
+ done.run(ebRequestProto.build());
+ } else {
+ controller.setFailed("Can't find query. request: " + request);
+ }
+ }
+
+ @Override
public void killQuery(RpcController controller, TajoIdProtos.QueryIdProto request,
RpcCallback<PrimitiveProtos.NullProto> done) {
QueryId queryId = new QueryId(request);
@@ -221,7 +228,7 @@ public class QueryMasterManagerService extends CompositeService
@Override
public void executeQuery(RpcController controller,
- TajoWorkerProtocol.QueryExecutionRequestProto request,
+ QueryExecutionRequest request,
RpcCallback<PrimitiveProtos.NullProto> done) {
workerContext.getWorkerSystemMetrics().counter("querymaster", "numQuery").inc();
@@ -231,7 +238,22 @@ public class QueryMasterManagerService extends CompositeService
new Session(request.getSession()),
new QueryContext(workerContext.getQueryMaster().getContext().getConf(),
request.getQueryContext()), request.getExprInJson().getValue(),
- request.getLogicalPlanJson().getValue()));
+ request.getLogicalPlanJson().getValue(), new NodeResource(request.getAllocation().getResource())));
done.run(TajoWorker.NULL_PROTO);
}
+
+ @Override
+ public void allocateQueryMaster(RpcController controller,
+ AllocationResourceProto request,
+ RpcCallback<PrimitiveProtos.BoolProto> done) {
+ CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>();
+ workerContext.getNodeResourceManager().handle(new QMResourceAllocateEvent(request, callFuture));
+
+ try {
+ done.run(callFuture.get());
+ } catch (Exception e) {
+ controller.setFailed(e.getMessage());
+ done.run(TajoWorker.FALSE_PROTO);
+ }
+ }
}