You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/12/19 13:48:49 UTC
[01/13] tajo git commit: TAJO-1252: PathValidator should allow hdfs
paths which contain IP addresses. (jihoon)
Repository: tajo
Updated Branches:
refs/heads/index_support 5893d2c98 -> 94faf6ec8
TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/e025e3cb
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/e025e3cb
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/e025e3cb
Branch: refs/heads/index_support
Commit: e025e3cbe80ab39ca963333a7d4d71edea22ce0b
Parents: 82dfb17
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Dec 17 16:17:25 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Dec 17 16:17:25 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../java/org/apache/tajo/validation/PathValidator.java | 2 +-
.../java/org/apache/tajo/validation/TestValidators.java | 12 ++++++++++++
3 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/e025e3cb/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 98c533b..65492d2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -109,6 +109,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses.
+ (jihoon)
+
TAJO-1246: HBase info port conflict occasionally causes unit test
failures in Jenkins CI. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/e025e3cb/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java
index 9548e80..34b5262 100644
--- a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java
+++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java
@@ -20,7 +20,7 @@ package org.apache.tajo.validation;
public class PathValidator extends PatternValidator {
static final String PATH_REGEXP_PATTERN =
- "^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$";
+ "^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z0-9-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$";
public PathValidator() {
super(PATH_REGEXP_PATTERN);
http://git-wip-us.apache.org/repos/asf/tajo/blob/e025e3cb/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java
index c539a51..92e016a 100644
--- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java
+++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java
@@ -384,6 +384,18 @@ public class TestValidators {
validUrl = "/tmp/tajo-root/";
assertThat(new PathValidator().validateInternal(validUrl), is(true));
assertThat(new PathValidator().validate(validUrl).size(), is(0));
+
+ validUrl = "hdfs://localhost:7020/tajo-root";
+ assertThat(new PathValidator().validateInternal(validUrl), is(true));
+ assertThat(new PathValidator().validate(validUrl).size(), is(0));
+
+ validUrl = "hdfs://127.0.0.1:7020/tajo-root";
+ assertThat(new PathValidator().validateInternal(validUrl), is(true));
+ assertThat(new PathValidator().validate(validUrl).size(), is(0));
+
+ validUrl = "hdfs://192.168.0.1:7020/tajo-root";
+ assertThat(new PathValidator().validateInternal(validUrl), is(true));
+ assertThat(new PathValidator().validate(validUrl).size(), is(0));
String invalidUrl = "t!ef:///tmp/tajo-root";
assertThat(new PathValidator().validateInternal(invalidUrl), is(false));
[04/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/worker/dataserver/TestHttpDataServer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/dataserver/TestHttpDataServer.java b/tajo-core/src/test/java/org/apache/tajo/worker/dataserver/TestHttpDataServer.java
index 2b3887f..25a2fbc 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/dataserver/TestHttpDataServer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/dataserver/TestHttpDataServer.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.worker.InterDataRetriever;
@@ -80,8 +80,8 @@ public class TestHttpDataServer {
public final void testInterDataRetriver() throws Exception {
MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
ExecutionBlockId schid = plan.newExecutionBlockId();
- QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
- QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);
+ TaskId qid1 = QueryIdFactory.newTaskId(schid);
+ TaskId qid2 = QueryIdFactory.newTaskId(schid);
File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
qid1Dir.mkdirs();
@@ -121,8 +121,8 @@ public class TestHttpDataServer {
public final void testNoSuchFile() throws Exception {
MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
ExecutionBlockId schid = plan.newExecutionBlockId();
- QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
- QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);
+ TaskId qid1 = QueryIdFactory.newTaskId(schid);
+ TaskId qid2 = QueryIdFactory.newTaskId(schid);
File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
qid1Dir.mkdirs();
@@ -154,7 +154,7 @@ public class TestHttpDataServer {
assertDataRetrival(qid1, addr.getPort(), watermark1);
}
- private static void assertDataRetrival(QueryUnitId id, int port,
+ private static void assertDataRetrival(TaskId id, int port,
String watermark) throws IOException {
URL url = new URL("http://127.0.0.1:"+port
+ "/?qid=" + id.toString() + "&fn=testHttp");
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case1.sql b/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case1.sql
deleted file mode 100644
index f6d0eb3..0000000
--- a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case1.sql
+++ /dev/null
@@ -1 +0,0 @@
-select l_linenumber, count(1) as unique_key from lineitem group by l_linenumber;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case2.sql b/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case2.sql
deleted file mode 100644
index c3e09f1..0000000
--- a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case2.sql
+++ /dev/null
@@ -1,5 +0,0 @@
-select l_returnflag, l_linestatus, sum(l_quantity) as sum_qty, sum(l_extendedprice) as sum_base_price, sum(l_extendedprice*(1-l_discount)) as sum_disc_price, sum
-(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge, avg(l_quantity) as avg_qty, avg(l_extendedprice) as avg_price, avg(l_discount) as avg_disc, count(*) as
- count_order
-from lineitem
-group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case3.sql b/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case3.sql
deleted file mode 100644
index a0f9c78..0000000
--- a/tajo-core/src/test/resources/queries/TestQueryUnitStatusUpdate/case3.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-select *
-from (
- select a.col1, a.col2, a.key
- from ColumnPartitionedTable a
- join ColumnPartitionedTable b on a.key = b.key
- where
- (a.key = 45.0 or a.key = 38.0)
-) test
-order by
- col1, col2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case1.sql b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case1.sql
new file mode 100644
index 0000000..f6d0eb3
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case1.sql
@@ -0,0 +1 @@
+select l_linenumber, count(1) as unique_key from lineitem group by l_linenumber;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case2.sql b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case2.sql
new file mode 100644
index 0000000..c3e09f1
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case2.sql
@@ -0,0 +1,5 @@
+select l_returnflag, l_linestatus, sum(l_quantity) as sum_qty, sum(l_extendedprice) as sum_base_price, sum(l_extendedprice*(1-l_discount)) as sum_disc_price, sum
+(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge, avg(l_quantity) as avg_qty, avg(l_extendedprice) as avg_price, avg(l_discount) as avg_disc, count(*) as
+ count_order
+from lineitem
+group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case3.sql b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case3.sql
new file mode 100644
index 0000000..a0f9c78
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTaskStatusUpdate/case3.sql
@@ -0,0 +1,10 @@
+select *
+from (
+ select a.col1, a.col2, a.key
+ from ColumnPartitionedTable a
+ join ColumnPartitionedTable b on a.key = b.key
+ where
+ (a.key = 45.0 or a.key = 38.0)
+) test
+order by
+ col1, col2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml
index c69c7d9..c408b16 100644
--- a/tajo-dist/pom.xml
+++ b/tajo-dist/pom.xml
@@ -225,8 +225,8 @@
<argument>Tajo</argument>
<argument>org.apache.tajo.master.querymaster.Query,
org.apache.tajo.master.querymaster.SubQuery,
- org.apache.tajo.master.querymaster.QueryUnit,
- org.apache.tajo.master.querymaster.QueryUnitAttempt
+ org.apache.tajo.master.querymaster.Task,
+ org.apache.tajo.master.querymaster.TaskAttempt
</argument>
<argument>Tajo.gv</argument>
</arguments>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java
----------------------------------------------------------------------
diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java
index 31db15c..bfb70b4 100644
--- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java
+++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java
@@ -232,7 +232,7 @@ public class HttpDataServerHandler extends SimpleChannelUpstreamHandler {
private List<String> splitMaps(List<String> qids) {
if (null == qids) {
- LOG.error("QueryUnitId is EMPTY");
+ LOG.error("QueryId is EMPTY");
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java
----------------------------------------------------------------------
diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java
index 67e7423..5591bba 100644
--- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java
+++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java
@@ -23,8 +23,8 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.pullserver.FileAccessForbiddenException;
import org.apache.tajo.util.TajoIdUtils;
import org.jboss.netty.channel.ChannelHandlerContext;
@@ -75,13 +75,13 @@ public class AdvancedDataRetriever implements DataRetriever {
if (params.containsKey("sid")) {
List<FileChunk> chunks = Lists.newArrayList();
- List<String> queryUnidIds = splitMaps(params.get("qid"));
- for (String eachQueryUnitId : queryUnidIds) {
- String[] queryUnitIdSeqTokens = eachQueryUnitId.split("_");
+ List<String> taskIds = splitMaps(params.get("qid"));
+ for (String eachTaskId : taskIds) {
+ String[] taskIdSeqTokens = eachTaskId.split("_");
ExecutionBlockId ebId = TajoIdUtils.createExecutionBlockId(params.get("sid").get(0));
- QueryUnitId quid = new QueryUnitId(ebId, Integer.parseInt(queryUnitIdSeqTokens[0]));
+ TaskId quid = new TaskId(ebId, Integer.parseInt(taskIdSeqTokens[0]));
- QueryUnitAttemptId attemptId = new QueryUnitAttemptId(quid, Integer.parseInt(queryUnitIdSeqTokens[1]));
+ TaskAttemptId attemptId = new TaskAttemptId(quid, Integer.parseInt(taskIdSeqTokens[1]));
RetrieverHandler handler = handlerMap.get(attemptId.toString());
FileChunk chunk = handler.get(params);
@@ -113,7 +113,7 @@ public class AdvancedDataRetriever implements DataRetriever {
private List<String> splitMaps(List<String> qids) {
if (null == qids) {
- LOG.error("QueryUnitId is EMPTY");
+ LOG.error("QueryId is EMPTY");
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index e2d89d6..07a51ba 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -68,7 +68,7 @@ public abstract class StorageManager {
private static final Class<?>[] DEFAULT_APPENDER_PARAMS = {
Configuration.class,
- QueryUnitAttemptId.class,
+ TaskAttemptId.class,
Schema.class,
TableMeta.class,
Path.class
@@ -446,7 +446,7 @@ public abstract class StorageManager {
* @throws java.io.IOException
*/
public Appender getAppender(OverridableConf queryContext,
- QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir)
+ TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir)
throws IOException {
Appender appender;
@@ -511,7 +511,7 @@ public abstract class StorageManager {
* @param <T>
* @return The scanner instance
*/
- public static <T> T newAppenderInstance(Class<T> theClass, Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public static <T> T newAppenderInstance(Class<T> theClass, Configuration conf, TaskAttemptId taskAttemptId,
TableMeta meta, Schema schema, Path workDir) {
T result;
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 2b196c9..af3d623 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -158,8 +158,8 @@ public class StorageUtil extends StorageConstants {
Path lastFile = fileNamePatternMatchedList.get(fileNamePatternMatchedList.size() - 1);
String pathName = lastFile.getName();
- // 0.8: pathName = part-<ExecutionBlockId.seq>-<QueryUnitId.seq>
- // 0.9: pathName = part-<ExecutionBlockId.seq>-<QueryUnitId.seq>-<Sequence>
+ // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>
+ // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence>
String[] pathTokens = pathName.split("-");
if (pathTokens.length == 3) {
return -1;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
index 8615235..4836352 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
@@ -21,7 +21,7 @@ package org.apache.tajo.storage.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -45,7 +45,7 @@ public abstract class AbstractHBaseAppender implements Appender {
protected Configuration conf;
protected Schema schema;
protected TableMeta meta;
- protected QueryUnitAttemptId taskAttemptId;
+ protected TaskAttemptId taskAttemptId;
protected Path stagingDir;
protected boolean inited = false;
@@ -72,7 +72,7 @@ public abstract class AbstractHBaseAppender implements Appender {
protected KeyValue[] keyValues;
- public AbstractHBaseAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public AbstractHBaseAppender(Configuration conf, TaskAttemptId taskAttemptId,
Schema schema, TableMeta meta, Path stagingDir) {
this.conf = conf;
this.schema = schema;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
index 50f61a8..45a1bff 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBasePutAppender.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -38,7 +38,7 @@ public class HBasePutAppender extends AbstractHBaseAppender {
private HTableInterface htable;
private long totalNumBytes;
- public HBasePutAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public HBasePutAppender(Configuration conf, TaskAttemptId taskAttemptId,
Schema schema, TableMeta meta, Path stagingDir) {
super(conf, taskAttemptId, schema, meta, stagingDir);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index a6e7a81..de4b4cb 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -539,7 +539,7 @@ public class HBaseStorageManager extends StorageManager {
@Override
public Appender getAppender(OverridableConf queryContext,
- QueryUnitAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir)
+ TaskAttemptId taskAttemptId, TableMeta meta, Schema schema, Path workDir)
throws IOException {
if ("true".equalsIgnoreCase(queryContext.get(HBaseStorageConstants.INSERT_PUT_MODE, "false"))) {
return new HBasePutAppender(conf, taskAttemptId, schema, meta, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java
index 07f7988..36678e4 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -53,7 +53,7 @@ public class HFileAppender extends AbstractHBaseAppender {
private Path workingFilePath;
private FileOutputCommitter committer;
- public HFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public HFileAppender(Configuration conf, TaskAttemptId taskAttemptId,
Schema schema, TableMeta meta, Path stagingDir) {
super(conf, taskAttemptId, schema, meta, stagingDir);
}
@@ -66,10 +66,10 @@ public class HFileAppender extends AbstractHBaseAppender {
Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);
taskConf.set(FileOutputFormat.OUTDIR, stagingResultDir.toString());
- ExecutionBlockId ebId = taskAttemptId.getQueryUnitId().getExecutionBlockId();
+ ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId();
writerContext = new TaskAttemptContextImpl(taskConf,
new TaskAttemptID(ebId.getQueryId().toString(), ebId.getId(), TaskType.MAP,
- taskAttemptId.getQueryUnitId().getId(), taskAttemptId.getId()));
+ taskAttemptId.getTaskId().getId(), taskAttemptId.getId()));
HFileOutputFormat2 hFileOutputFormat2 = new HFileOutputFormat2();
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
index 4bf4c99..28c263c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.*;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -76,7 +76,7 @@ public class CSVFile {
private NonSyncByteArrayOutputStream os = new NonSyncByteArrayOutputStream(BUFFER_SIZE);
private SerializerDeserializer serde;
- public CSVAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId,
+ public CSVAppender(Configuration conf, final TaskAttemptId taskAttemptId,
final Schema schema, final TableMeta meta, final Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
this.fs = workDir.getFileSystem(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java
index 47f67c6..b208a71 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileAppender.java
@@ -22,7 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
@@ -38,12 +38,12 @@ public abstract class FileAppender implements Appender {
protected final TableMeta meta;
protected final Schema schema;
protected final Path workDir;
- protected final QueryUnitAttemptId taskAttemptId;
+ protected final TaskAttemptId taskAttemptId;
protected boolean enabledStats;
protected Path path;
- public FileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema,
+ public FileAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema,
TableMeta meta, Path workDir) {
this.conf = conf;
this.meta = meta;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index 060bf16..3d4f7d5 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.tajo.OverridableConf;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -268,7 +268,7 @@ public class FileStorageManager extends StorageManager {
/////////////////////////////////////////////////////////////////////////////
// FileInputFormat Area
/////////////////////////////////////////////////////////////////////////////
- public Path getAppenderFilePath(QueryUnitAttemptId taskAttemptId, Path workDir) {
+ public Path getAppenderFilePath(TaskAttemptId taskAttemptId, Path workDir) {
if (taskAttemptId == null) {
// For testcase
return workDir;
@@ -277,8 +277,8 @@ public class FileStorageManager extends StorageManager {
// where ss is the subquery id associated with this task, and nnnnnn is the task id.
Path outFilePath = StorageUtil.concatPath(workDir, TajoConstants.RESULT_DIR_NAME,
OUTPUT_FILE_PREFIX +
- OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskAttemptId.getQueryUnitId().getExecutionBlockId().getId()) + "-" +
- OUTPUT_FILE_FORMAT_TASK.get().format(taskAttemptId.getQueryUnitId().getId()) + "-" +
+ OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskAttemptId.getTaskId().getExecutionBlockId().getId()) + "-" +
+ OUTPUT_FILE_FORMAT_TASK.get().format(taskAttemptId.getTaskId().getId()) + "-" +
OUTPUT_FILE_FORMAT_SEQ.get().format(0));
LOG.info("Output File Path: " + outFilePath);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java
index 33b2750..4c772c9 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java
@@ -21,7 +21,7 @@ package org.apache.tajo.storage;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.util.Pair;
@@ -42,7 +42,7 @@ public class HashShuffleAppender implements Appender {
private TableStats tableStats;
//<taskId,<page start offset,<task start, task end>>>
- private Map<QueryUnitAttemptId, List<Pair<Long, Pair<Integer, Integer>>>> taskTupleIndexes;
+ private Map<TaskAttemptId, List<Pair<Long, Pair<Integer, Integer>>>> taskTupleIndexes;
//page start offset, length
private List<Pair<Long, Integer>> pages = new ArrayList<Pair<Long, Integer>>();
@@ -69,7 +69,7 @@ public class HashShuffleAppender implements Appender {
@Override
public void init() throws IOException {
currentPage = new Pair(0L, 0);
- taskTupleIndexes = new HashMap<QueryUnitAttemptId, List<Pair<Long, Pair<Integer, Integer>>>>();
+ taskTupleIndexes = new HashMap<TaskAttemptId, List<Pair<Long, Pair<Integer, Integer>>>>();
rowNumInPage = 0;
}
@@ -81,7 +81,7 @@ public class HashShuffleAppender implements Appender {
* @return written bytes
* @throws java.io.IOException
*/
- public int addTuples(QueryUnitAttemptId taskId, List<Tuple> tuples) throws IOException {
+ public int addTuples(TaskAttemptId taskId, List<Tuple> tuples) throws IOException {
synchronized(appender) {
if (closed.get()) {
return 0;
@@ -189,7 +189,7 @@ public class HashShuffleAppender implements Appender {
return pages;
}
- public Map<QueryUnitAttemptId, List<Pair<Long, Pair<Integer, Integer>>>> getTaskTupleIndexes() {
+ public Map<TaskAttemptId, List<Pair<Long, Pair<Integer, Integer>>>> getTaskTupleIndexes() {
return taskTupleIndexes;
}
@@ -203,7 +203,7 @@ public class HashShuffleAppender implements Appender {
return merged;
}
- public void taskFinished(QueryUnitAttemptId taskId) {
+ public void taskFinished(TaskAttemptId taskId) {
taskTupleIndexes.remove(taskId);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
index 636ae0f..74190bc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
@@ -154,10 +154,10 @@ public class HashShuffleAppenderManager {
return intermEntries;
}
- public void finalizeTask(QueryUnitAttemptId taskId) {
+ public void finalizeTask(TaskAttemptId taskId) {
synchronized (appenderMap) {
Map<Integer, PartitionAppenderMeta> partitionAppenderMap =
- appenderMap.get(taskId.getQueryUnitId().getExecutionBlockId());
+ appenderMap.get(taskId.getTaskId().getExecutionBlockId());
if (partitionAppenderMap == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
index cb86f35..45e07d3 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -465,7 +465,7 @@ public class RawFile {
private TableStatistics stats;
- public RawFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public RawFileAppender(Configuration conf, TaskAttemptId taskAttemptId,
Schema schema, TableMeta meta, Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java
index 8da6ada..5510cbf 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -314,7 +314,7 @@ public class RowFile {
// statistics
private TableStatistics stats;
- public RowFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId,
+ public RowFileAppender(Configuration conf, final TaskAttemptId taskAttemptId,
final Schema schema, final TableMeta meta, final Path workDir)
throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java
index dbb8bd0..69399dc 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -61,7 +61,7 @@ public class AvroAppender extends FileAppender {
* @param workDir The path of the Parquet file to write to.
*/
public AvroAppender(Configuration conf,
- QueryUnitAttemptId taskAttemptId,
+ TaskAttemptId taskAttemptId,
org.apache.tajo.catalog.Schema schema,
TableMeta meta, Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java
index b10d423..ef5203c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java
@@ -18,7 +18,7 @@
package org.apache.tajo.storage.parquet;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.storage.StorageConstants;
import parquet.hadoop.ParquetOutputFormat;
import parquet.hadoop.metadata.CompressionCodecName;
@@ -54,7 +54,7 @@ public class ParquetAppender extends FileAppender {
* @param meta The table metadata.
* @param workDir The path of the Parquet file to write to.
*/
- public ParquetAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, TableMeta meta,
+ public ParquetAppender(Configuration conf, TaskAttemptId taskAttemptId, Schema schema, TableMeta meta,
Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
this.blockSize = Integer.parseInt(
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
index d88223b..2c09100 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.*;
import org.apache.hadoop.io.SequenceFile.Metadata;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -710,7 +710,7 @@ public class RCFile {
return out.getPos();
}
- public RCFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId,
+ public RCFileAppender(Configuration conf, final TaskAttemptId taskAttemptId,
final Schema schema, final TableMeta meta, final Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
index 14e0f26..8b5d677 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.io.*;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -74,7 +74,7 @@ public class SequenceFileAppender extends FileAppender {
private Writable EMPTY_KEY;
- public SequenceFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public SequenceFileAppender(Configuration conf, TaskAttemptId taskAttemptId,
Schema schema, TableMeta meta, Path workDir) throws IOException {
super(conf, taskAttemptId, schema, meta, workDir);
this.meta = meta;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index 8824e3e..59129d1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@ -30,14 +30,13 @@ import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.compress.CodecPool;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
-import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream;
import org.apache.tajo.util.ReflectionUtil;
@@ -114,7 +113,7 @@ public class DelimitedTextFile {
private NonSyncByteArrayOutputStream os;
private TextLineSerializer serializer;
- public DelimitedTextFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ public DelimitedTextFileAppender(Configuration conf, TaskAttemptId taskAttemptId,
final Schema schema, final TableMeta meta, final Path path)
throws IOException {
super(conf, taskAttemptId, schema, meta, path);
[07/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index 7f05fa4..d4cc6e7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -23,8 +23,6 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Event;
@@ -55,8 +53,8 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto;
import org.apache.tajo.master.*;
import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
-import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
+import org.apache.tajo.master.querymaster.Task.IntermediateEntry;
import org.apache.tajo.master.container.TajoContainer;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.FileStorageManager;
@@ -66,7 +64,7 @@ import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.util.history.QueryUnitHistory;
+import org.apache.tajo.util.history.TaskHistory;
import org.apache.tajo.util.history.SubQueryHistory;
import org.apache.tajo.worker.FetchImpl;
@@ -105,7 +103,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
private long startTime;
private long finishTime;
- volatile Map<QueryUnitId, QueryUnit> tasks = new ConcurrentHashMap<QueryUnitId, QueryUnit>();
+ volatile Map<TaskId, Task> tasks = new ConcurrentHashMap<TaskId, Task>();
volatile Map<TajoContainerId, TajoContainer> containers = new ConcurrentHashMap<TajoContainerId,
TajoContainer>();
@@ -355,22 +353,22 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
}
public float getProgress() {
- List<QueryUnit> tempTasks = null;
+ List<Task> tempTasks = null;
readLock.lock();
try {
if (getState() == SubQueryState.NEW) {
return 0.0f;
} else {
- tempTasks = new ArrayList<QueryUnit>(tasks.values());
+ tempTasks = new ArrayList<Task>(tasks.values());
}
} finally {
readLock.unlock();
}
float totalProgress = 0.0f;
- for (QueryUnit eachQueryUnit: tempTasks) {
- if (eachQueryUnit.getLastAttempt() != null) {
- totalProgress += eachQueryUnit.getLastAttempt().getProgress();
+ for (Task eachTask : tempTasks) {
+ if (eachTask.getLastAttempt() != null) {
+ totalProgress += eachTask.getLastAttempt().getProgress();
}
}
@@ -393,7 +391,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
return block;
}
- public void addTask(QueryUnit task) {
+ public void addTask(Task task) {
tasks.put(task.getId(), task);
}
@@ -401,7 +399,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
if (finalSubQueryHistory != null) {
if (finalSubQueryHistory.getFinishTime() == 0) {
finalSubQueryHistory = makeSubQueryHistory();
- finalSubQueryHistory.setQueryUnits(makeQueryUnitHistories());
+ finalSubQueryHistory.setTasks(makeTaskHistories());
}
return finalSubQueryHistory;
} else {
@@ -409,14 +407,14 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
}
}
- private List<QueryUnitHistory> makeQueryUnitHistories() {
- List<QueryUnitHistory> queryUnitHistories = new ArrayList<QueryUnitHistory>();
+ private List<TaskHistory> makeTaskHistories() {
+ List<TaskHistory> taskHistories = new ArrayList<TaskHistory>();
- for(QueryUnit eachQueryUnit: getQueryUnits()) {
- queryUnitHistories.add(eachQueryUnit.getQueryUnitHistory());
+ for(Task eachTask : getTasks()) {
+ taskHistories.add(eachTask.getTaskHistory());
}
- return queryUnitHistories;
+ return taskHistories;
}
private SubQueryHistory makeSubQueryHistory() {
@@ -440,16 +438,16 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
long totalWriteBytes = 0;
long totalWriteRows = 0;
int numShuffles = 0;
- for(QueryUnit eachQueryUnit: getQueryUnits()) {
- numShuffles = eachQueryUnit.getShuffleOutpuNum();
- if (eachQueryUnit.getLastAttempt() != null) {
- TableStats inputStats = eachQueryUnit.getLastAttempt().getInputStats();
+ for(Task eachTask : getTasks()) {
+ numShuffles = eachTask.getShuffleOutpuNum();
+ if (eachTask.getLastAttempt() != null) {
+ TableStats inputStats = eachTask.getLastAttempt().getInputStats();
if (inputStats != null) {
totalInputBytes += inputStats.getNumBytes();
totalReadBytes += inputStats.getReadBytes();
totalReadRows += inputStats.getNumRows();
}
- TableStats outputStats = eachQueryUnit.getLastAttempt().getResultStats();
+ TableStats outputStats = eachTask.getLastAttempt().getResultStats();
if (outputStats != null) {
totalWriteBytes += outputStats.getNumBytes();
totalWriteRows += outputStats.getNumRows();
@@ -511,11 +509,11 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
return block.getId();
}
- public QueryUnit[] getQueryUnits() {
- return tasks.values().toArray(new QueryUnit[tasks.size()]);
+ public Task[] getTasks() {
+ return tasks.values().toArray(new Task[tasks.size()]);
}
- public QueryUnit getQueryUnit(QueryUnitId qid) {
+ public Task getTask(TaskId qid) {
return tasks.get(qid);
}
@@ -635,7 +633,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
private TableStats[] computeStatFromTasks() {
List<TableStats> inputStatsList = Lists.newArrayList();
List<TableStats> resultStatsList = Lists.newArrayList();
- for (QueryUnit unit : getQueryUnits()) {
+ for (Task unit : getTasks()) {
resultStatsList.add(unit.getStats());
if (unit.getLastAttempt().getInputStats() != null) {
inputStatsList.add(unit.getLastAttempt().getInputStats());
@@ -1100,13 +1098,13 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
subQuery.getId(), fetches));
}
- public static QueryUnit newEmptyQueryUnit(TaskSchedulerContext schedulerContext,
- QueryUnitAttemptScheduleContext queryUnitContext,
- SubQuery subQuery, int taskId) {
+ public static Task newEmptyTask(TaskSchedulerContext schedulerContext,
+ TaskAttemptScheduleContext taskContext,
+ SubQuery subQuery, int taskId) {
ExecutionBlock execBlock = subQuery.getBlock();
- QueryUnit unit = new QueryUnit(schedulerContext.getMasterContext().getConf(),
- queryUnitContext,
- QueryIdFactory.newQueryUnitId(schedulerContext.getBlockId(), taskId),
+ Task unit = new Task(schedulerContext.getMasterContext().getConf(),
+ taskContext,
+ QueryIdFactory.newTaskId(schedulerContext.getBlockId(), taskId),
schedulerContext.isLeafQuery(), subQuery.eventHandler);
unit.setLogicalPlan(execBlock.getPlan());
subQuery.addTask(unit);
@@ -1176,7 +1174,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
public void transition(SubQuery subQuery,
SubQueryEvent event) {
SubQueryTaskEvent taskEvent = (SubQueryTaskEvent) event;
- QueryUnit task = subQuery.getQueryUnit(taskEvent.getTaskId());
+ Task task = subQuery.getTask(taskEvent.getTaskId());
if (task == null) { // task failed
LOG.error(String.format("Task %s is absent", taskEvent.getTaskId()));
@@ -1217,8 +1215,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
subQuery.getTaskScheduler().stop();
}
- for (QueryUnit queryUnit : subQuery.getQueryUnits()) {
- subQuery.eventHandler.handle(new TaskEvent(queryUnit.getId(), TaskEventType.T_KILL));
+ for (Task task : subQuery.getTasks()) {
+ subQuery.eventHandler.handle(new TaskEvent(task.getId(), TaskEventType.T_KILL));
}
}
}
@@ -1239,7 +1237,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
}
this.finalSubQueryHistory = makeSubQueryHistory();
- this.finalSubQueryHistory.setQueryUnits(makeQueryUnitHistories());
+ this.finalSubQueryHistory.setTasks(makeTaskHistories());
}
public List<IntermediateEntry> getHashShuffleIntermediateEntries() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Task.java
new file mode 100644
index 0000000..7de3933
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Task.java
@@ -0,0 +1,907 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.querymaster;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.state.*;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.QueryIdFactory;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TaskId;
+import org.apache.tajo.TajoProtos.TaskAttemptState;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.ipc.TajoWorkerProtocol.FailureIntermediateProto;
+import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto;
+import org.apache.tajo.master.FragmentPair;
+import org.apache.tajo.master.TaskState;
+import org.apache.tajo.master.event.*;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.storage.DataLocation;
+import org.apache.tajo.storage.fragment.FileFragment;
+import org.apache.tajo.storage.fragment.Fragment;
+import org.apache.tajo.storage.fragment.FragmentConvertor;
+import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.TajoIdUtils;
+import org.apache.tajo.util.history.TaskHistory;
+import org.apache.tajo.worker.FetchImpl;
+
+import java.net.URI;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
+import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
+
+public class Task implements EventHandler<TaskEvent> {
+ /** Class Logger */
+ private static final Log LOG = LogFactory.getLog(Task.class);
+
+ private final Configuration systemConf;
+ private TaskId taskId;
+ private EventHandler eventHandler;
+ private StoreTableNode store = null;
+ private LogicalNode plan = null;
+ private List<ScanNode> scan;
+
+ private Map<String, Set<FragmentProto>> fragMap;
+ private Map<String, Set<FetchImpl>> fetchMap;
+
+ private int totalFragmentNum;
+
+ private List<ShuffleFileOutput> shuffleFileOutputs;
+ private TableStats stats;
+ private final boolean isLeafTask;
+ private List<IntermediateEntry> intermediateData;
+
+ private Map<TaskAttemptId, TaskAttempt> attempts;
+ private final int maxAttempts = 3;
+ private Integer nextAttempt = -1;
+ private TaskAttemptId lastAttemptId;
+
+ private TaskAttemptId successfulAttempt;
+ private String succeededHost;
+ private int succeededHostPort;
+ private int succeededPullServerPort;
+
+ private int failedAttempts;
+ private int finishedAttempts; // finish are total of success, failed and killed
+
+ private long launchTime;
+ private long finishTime;
+
+ private List<DataLocation> dataLocations = Lists.newArrayList();
+
+ private static final AttemptKilledTransition ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
+
+ private TaskHistory finalTaskHistory;
+
+ protected static final StateMachineFactory
+ <Task, TaskState, TaskEventType, TaskEvent> stateMachineFactory =
+ new StateMachineFactory <Task, TaskState, TaskEventType, TaskEvent>(TaskState.NEW)
+
+ // Transitions from NEW state
+ .addTransition(TaskState.NEW, TaskState.SCHEDULED,
+ TaskEventType.T_SCHEDULE,
+ new InitialScheduleTransition())
+ .addTransition(TaskState.NEW, TaskState.KILLED,
+ TaskEventType.T_KILL,
+ new KillNewTaskTransition())
+
+ // Transitions from SCHEDULED state
+ .addTransition(TaskState.SCHEDULED, TaskState.RUNNING,
+ TaskEventType.T_ATTEMPT_LAUNCHED,
+ new AttemptLaunchedTransition())
+ .addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT,
+ TaskEventType.T_KILL,
+ new KillTaskTransition())
+
+ // Transitions from RUNNING state
+ .addTransition(TaskState.RUNNING, TaskState.RUNNING,
+ TaskEventType.T_ATTEMPT_LAUNCHED)
+ .addTransition(TaskState.RUNNING, TaskState.SUCCEEDED,
+ TaskEventType.T_ATTEMPT_SUCCEEDED,
+ new AttemptSucceededTransition())
+ .addTransition(TaskState.RUNNING, TaskState.KILL_WAIT,
+ TaskEventType.T_KILL,
+ new KillTaskTransition())
+ .addTransition(TaskState.RUNNING,
+ EnumSet.of(TaskState.RUNNING, TaskState.FAILED),
+ TaskEventType.T_ATTEMPT_FAILED,
+ new AttemptFailedOrRetryTransition())
+
+ // Transitions from KILL_WAIT state
+ .addTransition(TaskState.KILL_WAIT, TaskState.KILLED,
+ TaskEventType.T_ATTEMPT_KILLED,
+ ATTEMPT_KILLED_TRANSITION)
+ .addTransition(TaskState.KILL_WAIT, TaskState.KILL_WAIT,
+ TaskEventType.T_ATTEMPT_LAUNCHED,
+ new KillTaskTransition())
+ .addTransition(TaskState.KILL_WAIT, TaskState.FAILED,
+ TaskEventType.T_ATTEMPT_FAILED,
+ new AttemptFailedTransition())
+ .addTransition(TaskState.KILL_WAIT, TaskState.KILLED,
+ TaskEventType.T_ATTEMPT_SUCCEEDED,
+ ATTEMPT_KILLED_TRANSITION)
+ // Ignore-able transitions.
+ .addTransition(TaskState.KILL_WAIT, TaskState.KILL_WAIT,
+ EnumSet.of(
+ TaskEventType.T_KILL,
+ TaskEventType.T_SCHEDULE))
+
+ // Transitions from SUCCEEDED state
+ // Ignore-able transitions
+ .addTransition(TaskState.SUCCEEDED, TaskState.SUCCEEDED,
+ EnumSet.of(TaskEventType.T_KILL,
+ TaskEventType.T_ATTEMPT_KILLED, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
+
+ // Transitions from FAILED state
+ // Ignore-able transitions
+ .addTransition(TaskState.FAILED, TaskState.FAILED,
+ EnumSet.of(TaskEventType.T_KILL,
+ TaskEventType.T_ATTEMPT_KILLED, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
+
+ // Transitions from KILLED state
+ .addTransition(TaskState.KILLED, TaskState.KILLED, TaskEventType.T_ATTEMPT_KILLED, new KillTaskTransition())
+ // Ignore-able transitions
+ .addTransition(TaskState.KILLED, TaskState.KILLED,
+ EnumSet.of(
+ TaskEventType.T_KILL, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
+
+ .installTopology();
+
+ private final StateMachine<TaskState, TaskEventType, TaskEvent> stateMachine;
+
+
+ private final Lock readLock;
+ private final Lock writeLock;
+ private TaskAttemptScheduleContext scheduleContext;
+
+ public Task(Configuration conf, TaskAttemptScheduleContext scheduleContext,
+ TaskId id, boolean isLeafTask, EventHandler eventHandler) {
+ this.systemConf = conf;
+ this.taskId = id;
+ this.eventHandler = eventHandler;
+ this.isLeafTask = isLeafTask;
+ scan = new ArrayList<ScanNode>();
+ fetchMap = Maps.newHashMap();
+ fragMap = Maps.newHashMap();
+ shuffleFileOutputs = new ArrayList<ShuffleFileOutput>();
+ attempts = Collections.emptyMap();
+ lastAttemptId = null;
+ nextAttempt = -1;
+ failedAttempts = 0;
+
+ ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ this.readLock = readWriteLock.readLock();
+ this.writeLock = readWriteLock.writeLock();
+ this.scheduleContext = scheduleContext;
+
+ stateMachine = stateMachineFactory.make(this);
+ totalFragmentNum = 0;
+ }
+
+ public boolean isLeafTask() {
+ return this.isLeafTask;
+ }
+
+ public TaskState getState() {
+ readLock.lock();
+ try {
+ return stateMachine.getCurrentState();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public TaskAttemptState getLastAttemptStatus() {
+ TaskAttempt lastAttempt = getLastAttempt();
+ if (lastAttempt != null) {
+ return lastAttempt.getState();
+ } else {
+ return TaskAttemptState.TA_ASSIGNED;
+ }
+ }
+
+ public TaskHistory getTaskHistory() {
+ if (finalTaskHistory != null) {
+ if (finalTaskHistory.getFinishTime() == 0) {
+ finalTaskHistory = makeTaskHistory();
+ }
+ return finalTaskHistory;
+ } else {
+ return makeTaskHistory();
+ }
+ }
+
+ private TaskHistory makeTaskHistory() {
+ TaskHistory taskHistory = new TaskHistory();
+
+ TaskAttempt lastAttempt = getLastAttempt();
+ if (lastAttempt != null) {
+ taskHistory.setId(lastAttempt.getId().toString());
+ taskHistory.setState(lastAttempt.getState().toString());
+ taskHistory.setProgress(lastAttempt.getProgress());
+ }
+ taskHistory.setHostAndPort(succeededHost + ":" + succeededHostPort);
+ taskHistory.setRetryCount(this.getRetryCount());
+ taskHistory.setLaunchTime(launchTime);
+ taskHistory.setFinishTime(finishTime);
+
+ taskHistory.setNumShuffles(getShuffleOutpuNum());
+ if (!getShuffleFileOutputs().isEmpty()) {
+ ShuffleFileOutput shuffleFileOutputs = getShuffleFileOutputs().get(0);
+ if (taskHistory.getNumShuffles() > 0) {
+ taskHistory.setShuffleKey("" + shuffleFileOutputs.getPartId());
+ taskHistory.setShuffleFileName(shuffleFileOutputs.getFileName());
+ }
+ }
+
+ List<String> fragmentList = new ArrayList<String>();
+ for (FragmentProto eachFragment : getAllFragments()) {
+ try {
+ Fragment fragment = FragmentConvertor.convert(systemConf, eachFragment);
+ fragmentList.add(fragment.toString());
+ } catch (Exception e) {
+ LOG.error(e.getMessage());
+ fragmentList.add("ERROR: " + eachFragment.getStoreType() + "," + eachFragment.getId() + ": " + e.getMessage());
+ }
+ }
+ taskHistory.setFragments(fragmentList.toArray(new String[]{}));
+
+ List<String[]> fetchList = new ArrayList<String[]>();
+ for (Map.Entry<String, Set<FetchImpl>> e : getFetchMap().entrySet()) {
+ for (FetchImpl f : e.getValue()) {
+ for (URI uri : f.getSimpleURIs()){
+ fetchList.add(new String[] {e.getKey(), uri.toString()});
+ }
+ }
+ }
+
+ taskHistory.setFetchs(fetchList.toArray(new String[][]{}));
+
+ List<String> dataLocationList = new ArrayList<String>();
+ for(DataLocation eachLocation: getDataLocations()) {
+ dataLocationList.add(eachLocation.toString());
+ }
+
+ taskHistory.setDataLocations(dataLocationList.toArray(new String[]{}));
+ return taskHistory;
+ }
+
+ public void setLogicalPlan(LogicalNode plan) {
+ this.plan = plan;
+
+ LogicalNode node = plan;
+ ArrayList<LogicalNode> s = new ArrayList<LogicalNode>();
+ s.add(node);
+ while (!s.isEmpty()) {
+ node = s.remove(s.size()-1);
+ if (node instanceof UnaryNode) {
+ UnaryNode unary = (UnaryNode) node;
+ s.add(s.size(), unary.getChild());
+ } else if (node instanceof BinaryNode) {
+ BinaryNode binary = (BinaryNode) node;
+ s.add(s.size(), binary.getLeftChild());
+ s.add(s.size(), binary.getRightChild());
+ } else if (node instanceof ScanNode) {
+ scan.add((ScanNode)node);
+ } else if (node instanceof TableSubQueryNode) {
+ s.add(((TableSubQueryNode) node).getSubQuery());
+ }
+ }
+ }
+
+ private void addDataLocation(Fragment fragment) {
+ String[] hosts = fragment.getHosts();
+ int[] diskIds = null;
+ if (fragment instanceof FileFragment) {
+ diskIds = ((FileFragment)fragment).getDiskIds();
+ }
+ for (int i = 0; i < hosts.length; i++) {
+ dataLocations.add(new DataLocation(hosts[i], diskIds == null ? -1 : diskIds[i]));
+ }
+ }
+
+ public void addFragment(Fragment fragment, boolean useDataLocation) {
+ Set<FragmentProto> fragmentProtos;
+ if (fragMap.containsKey(fragment.getTableName())) {
+ fragmentProtos = fragMap.get(fragment.getTableName());
+ } else {
+ fragmentProtos = new HashSet<FragmentProto>();
+ fragMap.put(fragment.getTableName(), fragmentProtos);
+ }
+ fragmentProtos.add(fragment.getProto());
+ if (useDataLocation) {
+ addDataLocation(fragment);
+ }
+ totalFragmentNum++;
+ }
+
+ public void addFragments(Collection<Fragment> fragments) {
+ for (Fragment eachFragment: fragments) {
+ addFragment(eachFragment, false);
+ }
+ }
+
+ public void setFragment(FragmentPair[] fragmentPairs) {
+ for (FragmentPair eachFragmentPair : fragmentPairs) {
+ this.addFragment(eachFragmentPair.getLeftFragment(), true);
+ if (eachFragmentPair.getRightFragment() != null) {
+ this.addFragment(eachFragmentPair.getRightFragment(), true);
+ }
+ }
+ }
+
+ public List<DataLocation> getDataLocations() {
+ return dataLocations;
+ }
+
+ public String getSucceededHost() {
+ return succeededHost;
+ }
+
+ public void addFetches(String tableId, Collection<FetchImpl> fetches) {
+ Set<FetchImpl> fetchSet;
+ if (fetchMap.containsKey(tableId)) {
+ fetchSet = fetchMap.get(tableId);
+ } else {
+ fetchSet = Sets.newHashSet();
+ }
+ fetchSet.addAll(fetches);
+ fetchMap.put(tableId, fetchSet);
+ }
+
+ public void setFetches(Map<String, Set<FetchImpl>> fetches) {
+ this.fetchMap.clear();
+ this.fetchMap.putAll(fetches);
+ }
+
+ public Collection<FragmentProto> getAllFragments() {
+ Set<FragmentProto> fragmentProtos = new HashSet<FragmentProto>();
+ for (Set<FragmentProto> eachFragmentSet : fragMap.values()) {
+ fragmentProtos.addAll(eachFragmentSet);
+ }
+ return fragmentProtos;
+ }
+
+ public LogicalNode getLogicalPlan() {
+ return this.plan;
+ }
+
+ public TaskId getId() {
+ return taskId;
+ }
+
+ public Collection<FetchImpl> getFetchHosts(String tableId) {
+ return fetchMap.get(tableId);
+ }
+
+ public Collection<Set<FetchImpl>> getFetches() {
+ return fetchMap.values();
+ }
+
+ public Map<String, Set<FetchImpl>> getFetchMap() {
+ return fetchMap;
+ }
+
+ public Collection<FetchImpl> getFetch(ScanNode scan) {
+ return this.fetchMap.get(scan.getTableName());
+ }
+
+ public ScanNode[] getScanNodes() {
+ return this.scan.toArray(new ScanNode[scan.size()]);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append(plan.getType() + " \n");
+ for (Entry<String, Set<FragmentProto>> e : fragMap.entrySet()) {
+ builder.append(e.getKey()).append(" : ");
+ for (FragmentProto fragment : e.getValue()) {
+ builder.append(fragment).append(", ");
+ }
+ }
+ for (Entry<String, Set<FetchImpl>> e : fetchMap.entrySet()) {
+ builder.append(e.getKey()).append(" : ");
+ for (FetchImpl t : e.getValue()) {
+ for (URI uri : t.getURIs()){
+ builder.append(uri).append(" ");
+ }
+ }
+ }
+
+ return builder.toString();
+ }
+
+ public void setStats(TableStats stats) {
+ this.stats = stats;
+ }
+
+ public void setShuffleFileOutputs(List<ShuffleFileOutput> partitions) {
+ this.shuffleFileOutputs = Collections.unmodifiableList(partitions);
+ }
+
+ public TableStats getStats() {
+ return this.stats;
+ }
+
+ public List<ShuffleFileOutput> getShuffleFileOutputs() {
+ return this.shuffleFileOutputs;
+ }
+
+ public int getShuffleOutpuNum() {
+ return this.shuffleFileOutputs.size();
+ }
+
+ public TaskAttempt newAttempt() {
+ TaskAttempt attempt = new TaskAttempt(scheduleContext,
+ QueryIdFactory.newTaskAttemptId(this.getId(), ++nextAttempt),
+ this, eventHandler);
+ lastAttemptId = attempt.getId();
+ return attempt;
+ }
+
+ public TaskAttempt getAttempt(TaskAttemptId attemptId) {
+ return attempts.get(attemptId);
+ }
+
+ public TaskAttempt getAttempt(int attempt) {
+ return this.attempts.get(QueryIdFactory.newTaskAttemptId(this.getId(), attempt));
+ }
+
+ public TaskAttempt getLastAttempt() {
+ return getAttempt(this.lastAttemptId);
+ }
+
+ public TaskAttempt getSuccessfulAttempt() {
+ readLock.lock();
+ try {
+ if (null == successfulAttempt) {
+ return null;
+ }
+ return attempts.get(successfulAttempt);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public int getRetryCount () {
+ return this.nextAttempt;
+ }
+
+ public int getTotalFragmentNum() {
+ return totalFragmentNum;
+ }
+
+ private static class InitialScheduleTransition implements
+ SingleArcTransition<Task, TaskEvent> {
+
+ @Override
+ public void transition(Task task, TaskEvent taskEvent) {
+ task.addAndScheduleAttempt();
+ }
+ }
+
+ public long getLaunchTime() {
+ return launchTime;
+ }
+
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ @VisibleForTesting
+ public void setLaunchTime(long launchTime) {
+ this.launchTime = launchTime;
+ }
+
+ @VisibleForTesting
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ public long getRunningTime() {
+ if(finishTime > 0) {
+ return finishTime - launchTime;
+ } else {
+ return System.currentTimeMillis() - launchTime;
+ }
+ }
+
+ // This is always called in the Write Lock
+ private void addAndScheduleAttempt() {
+ // Create new task attempt
+ TaskAttempt attempt = newAttempt();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created attempt " + attempt.getId());
+ }
+ switch (attempts.size()) {
+ case 0:
+ attempts = Collections.singletonMap(attempt.getId(), attempt);
+ break;
+
+ case 1:
+ Map<TaskAttemptId, TaskAttempt> newAttempts
+ = new LinkedHashMap<TaskAttemptId, TaskAttempt>(3);
+ newAttempts.putAll(attempts);
+ attempts = newAttempts;
+ attempts.put(attempt.getId(), attempt);
+ break;
+
+ default:
+ attempts.put(attempt.getId(), attempt);
+ break;
+ }
+
+ if (failedAttempts > 0) {
+ eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
+ TaskAttemptEventType.TA_RESCHEDULE));
+ } else {
+ eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
+ TaskAttemptEventType.TA_SCHEDULE));
+ }
+ }
+
+ private void finishTask() {
+ this.finishTime = System.currentTimeMillis();
+ finalTaskHistory = makeTaskHistory();
+ }
+
+ private static class KillNewTaskTransition implements SingleArcTransition<Task, TaskEvent> {
+
+ @Override
+ public void transition(Task task, TaskEvent taskEvent) {
+ task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.KILLED));
+ }
+ }
+
+ private static class KillTaskTransition implements SingleArcTransition<Task, TaskEvent> {
+
+ @Override
+ public void transition(Task task, TaskEvent taskEvent) {
+ task.finishTask();
+ task.eventHandler.handle(new TaskAttemptEvent(task.lastAttemptId, TaskAttemptEventType.TA_KILL));
+ }
+ }
+
+ private static class AttemptKilledTransition implements SingleArcTransition<Task, TaskEvent>{
+
+ @Override
+ public void transition(Task task, TaskEvent event) {
+ task.finishTask();
+ task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.KILLED));
+ }
+ }
+
+ private static class AttemptSucceededTransition
+ implements SingleArcTransition<Task, TaskEvent>{
+
+ @Override
+ public void transition(Task task,
+ TaskEvent event) {
+ TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
+ TaskAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
+
+ task.successfulAttempt = attemptEvent.getTaskAttemptId();
+ task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
+ task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
+ task.succeededPullServerPort = attempt.getWorkerConnectionInfo().getPullServerPort();
+
+ task.finishTask();
+ task.eventHandler.handle(new SubQueryTaskEvent(event.getTaskId(), TaskState.SUCCEEDED));
+ }
+ }
+
+ private static class AttemptLaunchedTransition implements SingleArcTransition<Task, TaskEvent> {
+ @Override
+ public void transition(Task task,
+ TaskEvent event) {
+ TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
+ TaskAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
+ task.launchTime = System.currentTimeMillis();
+ task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
+ task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
+ }
+ }
+
+ private static class AttemptFailedTransition implements SingleArcTransition<Task, TaskEvent> {
+ @Override
+ public void transition(Task task, TaskEvent event) {
+ TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
+ LOG.info("=============================================================");
+ LOG.info(">>> Task Failed: " + attemptEvent.getTaskAttemptId() + " <<<");
+ LOG.info("=============================================================");
+ task.failedAttempts++;
+ task.finishedAttempts++;
+
+ task.finishTask();
+ task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.FAILED));
+ }
+ }
+
+ private static class AttemptFailedOrRetryTransition implements
+ MultipleArcTransition<Task, TaskEvent, TaskState> {
+
+ @Override
+ public TaskState transition(Task task, TaskEvent taskEvent) {
+ TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) taskEvent;
+ task.failedAttempts++;
+ task.finishedAttempts++;
+ boolean retry = task.failedAttempts < task.maxAttempts;
+
+ LOG.info("====================================================================================");
+ LOG.info(">>> Task Failed: " + attemptEvent.getTaskAttemptId() + ", " +
+ "retry:" + retry + ", attempts:" + task.failedAttempts + " <<<");
+ LOG.info("====================================================================================");
+
+ if (retry) {
+ if (task.successfulAttempt == null) {
+ task.addAndScheduleAttempt();
+ }
+ } else {
+ task.finishTask();
+ task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.FAILED));
+ return TaskState.FAILED;
+ }
+
+ return task.getState();
+ }
+ }
+
+ @Override
+ public void handle(TaskEvent event) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing " + event.getTaskId() + " of type "
+ + event.getType());
+ }
+
+ try {
+ writeLock.lock();
+ TaskState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitonException e) {
+ LOG.error("Can't handle this event at current state"
+ + ", eventType:" + event.getType().name()
+ + ", oldState:" + oldState.name()
+ + ", nextState:" + getState().name()
+ , e);
+ eventHandler.handle(new QueryEvent(TajoIdUtils.parseQueryId(getId().toString()),
+ QueryEventType.INTERNAL_ERROR));
+ }
+
+ //notify the eventhandler of state change
+ if (LOG.isDebugEnabled()) {
+ if (oldState != getState()) {
+ LOG.debug(taskId + " Task Transitioned from " + oldState + " to "
+ + getState());
+ }
+ }
+ }
+
+ finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void setIntermediateData(Collection<IntermediateEntry> partitions) {
+ this.intermediateData = new ArrayList<IntermediateEntry>(partitions);
+ }
+
+ public List<IntermediateEntry> getIntermediateData() {
+ return this.intermediateData;
+ }
+
+ public static class PullHost implements Cloneable {
+ String host;
+ int port;
+ int hashCode;
+
+ public PullHost(String pullServerAddr, int pullServerPort){
+ this.host = pullServerAddr;
+ this.port = pullServerPort;
+ this.hashCode = Objects.hashCode(host, port);
+ }
+ public String getHost() {
+ return host;
+ }
+
+ public int getPort() {
+ return this.port;
+ }
+
+ public String getPullAddress() {
+ return host + ":" + port;
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof PullHost) {
+ PullHost other = (PullHost) obj;
+ return host.equals(other.host) && port == other.port;
+ }
+
+ return false;
+ }
+
+ @Override
+ public PullHost clone() throws CloneNotSupportedException {
+ PullHost newPullHost = (PullHost) super.clone();
+ newPullHost.host = host;
+ newPullHost.port = port;
+ newPullHost.hashCode = Objects.hashCode(newPullHost.host, newPullHost.port);
+ return newPullHost;
+ }
+
+ @Override
+ public String toString() {
+ return host + ":" + port;
+ }
+ }
+
+ public static class IntermediateEntry {
+ ExecutionBlockId ebId;
+ int taskId;
+ int attemptId;
+ int partId;
+ PullHost host;
+ long volume;
+ List<Pair<Long, Integer>> pages;
+ List<Pair<Long, Pair<Integer, Integer>>> failureRowNums;
+
+ public IntermediateEntry(IntermediateEntryProto proto) {
+ this.ebId = new ExecutionBlockId(proto.getEbId());
+ this.taskId = proto.getTaskId();
+ this.attemptId = proto.getAttemptId();
+ this.partId = proto.getPartId();
+
+ String[] pullHost = proto.getHost().split(":");
+ this.host = new PullHost(pullHost[0], Integer.parseInt(pullHost[1]));
+ this.volume = proto.getVolume();
+
+ failureRowNums = new ArrayList<Pair<Long, Pair<Integer, Integer>>>();
+ for (FailureIntermediateProto eachFailure: proto.getFailuresList()) {
+
+ failureRowNums.add(new Pair(eachFailure.getPagePos(),
+ new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum())));
+ }
+
+ pages = new ArrayList<Pair<Long, Integer>>();
+ for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) {
+ pages.add(new Pair(eachPage.getPos(), eachPage.getLength()));
+ }
+ }
+
+ public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) {
+ this.taskId = taskId;
+ this.attemptId = attemptId;
+ this.partId = partId;
+ this.host = host;
+ }
+
+ public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host, long volume) {
+ this.taskId = taskId;
+ this.attemptId = attemptId;
+ this.partId = partId;
+ this.host = host;
+ this.volume = volume;
+ }
+
+ public ExecutionBlockId getEbId() {
+ return ebId;
+ }
+
+ public void setEbId(ExecutionBlockId ebId) {
+ this.ebId = ebId;
+ }
+
+ public int getTaskId() {
+ return this.taskId;
+ }
+
+ public int getAttemptId() {
+ return this.attemptId;
+ }
+
+ public int getPartId() {
+ return this.partId;
+ }
+
+ public PullHost getPullHost() {
+ return this.host;
+ }
+
+ public long getVolume() {
+ return this.volume;
+ }
+
+ public long setVolume(long volume) {
+ return this.volume = volume;
+ }
+
+ public List<Pair<Long, Integer>> getPages() {
+ return pages;
+ }
+
+ public void setPages(List<Pair<Long, Integer>> pages) {
+ this.pages = pages;
+ }
+
+ public List<Pair<Long, Pair<Integer, Integer>>> getFailureRowNums() {
+ return failureRowNums;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(ebId, taskId, partId, attemptId, host);
+ }
+
+ public List<Pair<Long, Long>> split(long firstSplitVolume, long splitVolume) {
+ List<Pair<Long, Long>> splits = new ArrayList<Pair<Long, Long>>();
+
+ if (pages == null || pages.isEmpty()) {
+ return splits;
+ }
+ int pageSize = pages.size();
+
+ long currentOffset = -1;
+ long currentBytes = 0;
+
+ long realSplitVolume = firstSplitVolume > 0 ? firstSplitVolume : splitVolume;
+ for (int i = 0; i < pageSize; i++) {
+ Pair<Long, Integer> eachPage = pages.get(i);
+ if (currentOffset == -1) {
+ currentOffset = eachPage.getFirst();
+ }
+ if (currentBytes > 0 && currentBytes + eachPage.getSecond() >= realSplitVolume) {
+ splits.add(new Pair(currentOffset, currentBytes));
+ currentOffset = eachPage.getFirst();
+ currentBytes = 0;
+ realSplitVolume = splitVolume;
+ }
+
+ currentBytes += eachPage.getSecond();
+ }
+
+ //add last
+ if (currentBytes > 0) {
+ splits.add(new Pair(currentOffset, currentBytes));
+ }
+ return splits;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/TaskAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/TaskAttempt.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/TaskAttempt.java
new file mode 100644
index 0000000..63c6dbb
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/TaskAttempt.java
@@ -0,0 +1,443 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.querymaster;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.state.*;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TajoProtos.TaskAttemptState;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
+import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.event.*;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
+import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
+import org.apache.tajo.master.querymaster.Task.IntermediateEntry;
+import org.apache.tajo.master.querymaster.Task.PullHost;
+import org.apache.tajo.master.container.TajoContainerId;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
+
+public class TaskAttempt implements EventHandler<TaskAttemptEvent> {
+
+ private static final Log LOG = LogFactory.getLog(TaskAttempt.class);
+
+ private final static int EXPIRE_TIME = 15000;
+
+ private final TaskAttemptId id;
+ private final Task task;
+ final EventHandler eventHandler;
+
+ private TajoContainerId containerId;
+ private WorkerConnectionInfo workerConnectionInfo;
+ private int expire;
+
+ private final Lock readLock;
+ private final Lock writeLock;
+
+ private final List<String> diagnostics = new ArrayList<String>();
+
+ private final TaskAttemptScheduleContext scheduleContext;
+
+ private float progress;
+ private CatalogProtos.TableStatsProto inputStats;
+ private CatalogProtos.TableStatsProto resultStats;
+
+ protected static final StateMachineFactory
+ <TaskAttempt, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+ stateMachineFactory = new StateMachineFactory
+ <TaskAttempt, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+ (TaskAttemptState.TA_NEW)
+
+ // Transitions from TA_NEW state
+ .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_UNASSIGNED,
+ TaskAttemptEventType.TA_SCHEDULE, new TaskAttemptScheduleTransition())
+ .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_UNASSIGNED,
+ TaskAttemptEventType.TA_RESCHEDULE, new TaskAttemptScheduleTransition())
+ .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_KILL,
+ new TaskKilledCompleteTransition())
+
+ // Transitions from TA_UNASSIGNED state
+ .addTransition(TaskAttemptState.TA_UNASSIGNED, TaskAttemptState.TA_ASSIGNED,
+ TaskAttemptEventType.TA_ASSIGNED,
+ new LaunchTransition())
+ .addTransition(TaskAttemptState.TA_UNASSIGNED, TaskAttemptState.TA_KILL_WAIT,
+ TaskAttemptEventType.TA_KILL,
+ new KillUnassignedTaskTransition())
+
+ // Transitions from TA_ASSIGNED state
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_ASSIGNED,
+ TaskAttemptEventType.TA_ASSIGNED, new AlreadyAssignedTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_KILL_WAIT,
+ TaskAttemptEventType.TA_KILL,
+ new KillTaskTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_KILL,
+ new KillTaskTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED,
+ EnumSet.of(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_KILLED),
+ TaskAttemptEventType.TA_UPDATE, new StatusUpdateTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_SUCCEEDED,
+ TaskAttemptEventType.TA_DONE, new SucceededTransition())
+ .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_FAILED,
+ TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
+
+ // Transitions from TA_RUNNING state
+ .addTransition(TaskAttemptState.TA_RUNNING,
+ EnumSet.of(TaskAttemptState.TA_RUNNING),
+ TaskAttemptEventType.TA_UPDATE, new StatusUpdateTransition())
+ .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_KILL_WAIT,
+ TaskAttemptEventType.TA_KILL,
+ new KillTaskTransition())
+ .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_SUCCEEDED,
+ TaskAttemptEventType.TA_DONE, new SucceededTransition())
+ .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_FAILED,
+ TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
+
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_LOCAL_KILLED,
+ new TaskKilledCompleteTransition())
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILL_WAIT,
+ TaskAttemptEventType.TA_ASSIGNED,
+ new KillTaskTransition())
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_SCHEDULE_CANCELED,
+ new TaskKilledCompleteTransition())
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_DONE,
+ new TaskKilledCompleteTransition())
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_FAILED,
+ TaskAttemptEventType.TA_FATAL_ERROR)
+ .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILL_WAIT,
+ EnumSet.of(
+ TaskAttemptEventType.TA_KILL,
+ TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
+ TaskAttemptEventType.TA_UPDATE))
+
+ // Transitions from TA_SUCCEEDED state
+ .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
+ TaskAttemptEventType.TA_UPDATE)
+ .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
+ TaskAttemptEventType.TA_DONE, new AlreadyDoneTransition())
+ .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_FAILED,
+ TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
+ // Ignore-able transitions
+ .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
+ TaskAttemptEventType.TA_KILL)
+
+ // Transitions from TA_KILLED state
+ .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
+ TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE)
+ // Ignore-able transitions
+ .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
+ EnumSet.of(
+ TaskAttemptEventType.TA_UPDATE))
+ .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
+ EnumSet.of(
+ TaskAttemptEventType.TA_LOCAL_KILLED,
+ TaskAttemptEventType.TA_KILL,
+ TaskAttemptEventType.TA_ASSIGNED,
+ TaskAttemptEventType.TA_DONE),
+ new TaskKilledCompleteTransition())
+ .installTopology();
+
+ private final StateMachine<TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
+ stateMachine;
+
+
+ public TaskAttempt(final TaskAttemptScheduleContext scheduleContext,
+ final TaskAttemptId id, final Task task,
+ final EventHandler eventHandler) {
+ this.scheduleContext = scheduleContext;
+ this.id = id;
+ this.expire = TaskAttempt.EXPIRE_TIME;
+ this.task = task;
+ this.eventHandler = eventHandler;
+
+ ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ this.readLock = readWriteLock.readLock();
+ this.writeLock = readWriteLock.writeLock();
+
+ stateMachine = stateMachineFactory.make(this);
+ }
+
+ public TaskAttemptState getState() {
+ readLock.lock();
+ try {
+ return stateMachine.getCurrentState();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public TaskAttemptId getId() {
+ return this.id;
+ }
+
+ public boolean isLeafTask() {
+ return this.task.isLeafTask();
+ }
+
+ public Task getTask() {
+ return this.task;
+ }
+
+ public WorkerConnectionInfo getWorkerConnectionInfo() {
+ return this.workerConnectionInfo;
+ }
+
+ public void setContainerId(TajoContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ public synchronized void setExpireTime(int expire) {
+ this.expire = expire;
+ }
+
+ public synchronized void updateExpireTime(int period) {
+ this.setExpireTime(this.expire - period);
+ }
+
+ public synchronized void resetExpireTime() {
+ this.setExpireTime(TaskAttempt.EXPIRE_TIME);
+ }
+
+ public int getLeftTime() {
+ return this.expire;
+ }
+
+ public float getProgress() {
+ return progress;
+ }
+
+ public TableStats getInputStats() {
+ if (inputStats == null) {
+ return null;
+ }
+
+ return new TableStats(inputStats);
+ }
+
+ public TableStats getResultStats() {
+ if (resultStats == null) {
+ return null;
+ }
+ return new TableStats(resultStats);
+ }
+
+ private void fillTaskStatistics(TaskCompletionReport report) {
+ this.progress = 1.0f;
+
+ List<IntermediateEntry> partitions = new ArrayList<IntermediateEntry>();
+
+ if (report.getShuffleFileOutputsCount() > 0) {
+ this.getTask().setShuffleFileOutputs(report.getShuffleFileOutputsList());
+
+ PullHost host = new PullHost(getWorkerConnectionInfo().getHost(), getWorkerConnectionInfo().getPullServerPort());
+ for (ShuffleFileOutput p : report.getShuffleFileOutputsList()) {
+ IntermediateEntry entry = new IntermediateEntry(getId().getTaskId().getId(),
+ getId().getId(), p.getPartId(), host, p.getVolume());
+ partitions.add(entry);
+ }
+ }
+ this.getTask().setIntermediateData(partitions);
+
+ if (report.hasInputStats()) {
+ this.inputStats = report.getInputStats();
+ }
+ if (report.hasResultStats()) {
+ this.resultStats = report.getResultStats();
+ this.getTask().setStats(new TableStats(resultStats));
+ }
+ }
+
+ private static class TaskAttemptScheduleTransition implements
+ SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt, TaskAttemptEvent taskAttemptEvent) {
+ taskAttempt.eventHandler.handle(new TaskAttemptToSchedulerEvent(
+ EventType.T_SCHEDULE, taskAttempt.getTask().getId().getExecutionBlockId(),
+ taskAttempt.scheduleContext, taskAttempt));
+ }
+ }
+
+ private static class KillUnassignedTaskTransition implements
+ SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt, TaskAttemptEvent taskAttemptEvent) {
+ taskAttempt.eventHandler.handle(new TaskAttemptToSchedulerEvent(
+ EventType.T_SCHEDULE_CANCEL, taskAttempt.getTask().getId().getExecutionBlockId(),
+ taskAttempt.scheduleContext, taskAttempt));
+ }
+ }
+
+ private static class LaunchTransition
+ implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent event) {
+ TaskAttemptAssignedEvent castEvent = (TaskAttemptAssignedEvent) event;
+ taskAttempt.containerId = castEvent.getContainerId();
+ taskAttempt.workerConnectionInfo = castEvent.getWorkerConnectionInfo();
+ taskAttempt.eventHandler.handle(
+ new TaskTAttemptEvent(taskAttempt.getId(),
+ TaskEventType.T_ATTEMPT_LAUNCHED));
+ }
+ }
+
+ private static class TaskKilledCompleteTransition implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent event) {
+ taskAttempt.getTask().handle(new TaskEvent(taskAttempt.getId().getTaskId(),
+ TaskEventType.T_ATTEMPT_KILLED));
+ LOG.info(taskAttempt.getId() + " Received TA_KILLED Status from LocalTask");
+ }
+ }
+
+ private static class StatusUpdateTransition
+ implements MultipleArcTransition<TaskAttempt, TaskAttemptEvent, TaskAttemptState> {
+
+ @Override
+ public TaskAttemptState transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent event) {
+ TaskAttemptStatusUpdateEvent updateEvent = (TaskAttemptStatusUpdateEvent) event;
+
+ taskAttempt.progress = updateEvent.getStatus().getProgress();
+ taskAttempt.inputStats = updateEvent.getStatus().getInputStats();
+ taskAttempt.resultStats = updateEvent.getStatus().getResultStats();
+
+ return TaskAttemptState.TA_RUNNING;
+ }
+ }
+
+ private void addDiagnosticInfo(String diag) {
+ if (diag != null && !diag.equals("")) {
+ diagnostics.add(diag);
+ }
+ }
+
+ private static class AlreadyAssignedTransition
+ implements SingleArcTransition<TaskAttempt, TaskAttemptEvent>{
+
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent taskAttemptEvent) {
+ }
+ }
+
+ private static class AlreadyDoneTransition
+ implements SingleArcTransition<TaskAttempt, TaskAttemptEvent>{
+
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent taskAttemptEvent) {
+ }
+ }
+
+ private static class SucceededTransition implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+ @Override
+ public void transition(TaskAttempt taskAttempt,
+ TaskAttemptEvent event) {
+ TaskCompletionReport report = ((TaskCompletionEvent)event).getReport();
+
+ try {
+ taskAttempt.fillTaskStatistics(report);
+ taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
+ } catch (Throwable t) {
+ taskAttempt.eventHandler.handle(new TaskFatalErrorEvent(taskAttempt.getId(), t.getMessage()));
+ taskAttempt.addDiagnosticInfo(ExceptionUtils.getStackTrace(t));
+ }
+ }
+ }
+
+ private static class KillTaskTransition implements SingleArcTransition<TaskAttempt, TaskAttemptEvent> {
+
+ @Override
+ public void transition(TaskAttempt taskAttempt, TaskAttemptEvent event) {
+ taskAttempt.eventHandler.handle(new LocalTaskEvent(taskAttempt.getId(), taskAttempt.containerId,
+ LocalTaskEventType.KILL));
+ }
+ }
+
+ private static class FailedTransition implements SingleArcTransition<TaskAttempt, TaskAttemptEvent>{
+ @Override
+ public void transition(TaskAttempt taskAttempt, TaskAttemptEvent event) {
+ TaskFatalErrorEvent errorEvent = (TaskFatalErrorEvent) event;
+ taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_FAILED));
+ taskAttempt.addDiagnosticInfo(errorEvent.errorMessage());
+ LOG.error(taskAttempt.getId() + " FROM " + taskAttempt.getWorkerConnectionInfo().getHost()
+ + " >> " + errorEvent.errorMessage());
+ }
+ }
+
+ @Override
+ public void handle(TaskAttemptEvent event) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing " + event.getTaskAttemptId() + " of type " + event.getType());
+ }
+ try {
+ writeLock.lock();
+ TaskAttemptState oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitonException e) {
+ LOG.error("Can't handle this event at current state of " + event.getTaskAttemptId() + ")"
+ + ", eventType:" + event.getType().name()
+ + ", oldState:" + oldState.name()
+ + ", nextState:" + getState().name()
+ , e);
+ eventHandler.handle(
+ new SubQueryDiagnosticsUpdateEvent(event.getTaskAttemptId().getTaskId().getExecutionBlockId(),
+ "Can't handle this event at current state of " + event.getTaskAttemptId() + ")"));
+ eventHandler.handle(
+ new SubQueryEvent(event.getTaskAttemptId().getTaskId().getExecutionBlockId(),
+ SubQueryEventType.SQ_INTERNAL_ERROR));
+ }
+
+ //notify the eventhandler of state change
+ if (LOG.isDebugEnabled()) {
+ if (oldState != getState()) {
+ LOG.debug(id + " TaskAttempt Transitioned from " + oldState + " to "
+ + getState());
+ }
+ }
+ }
+
+ finally {
+ writeLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
index 6d3597d..0573197 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java
@@ -27,9 +27,9 @@ import org.apache.tajo.master.TajoMaster.MasterContext;
import org.apache.tajo.master.ha.HAService;
import org.apache.tajo.master.querymaster.QueryInProgress;
import org.apache.tajo.master.querymaster.QueryMasterTask;
-import org.apache.tajo.master.querymaster.QueryUnit;
+import org.apache.tajo.master.querymaster.Task;
import org.apache.tajo.master.querymaster.SubQuery;
-import org.apache.tajo.util.history.QueryUnitHistory;
+import org.apache.tajo.util.history.TaskHistory;
import org.apache.tajo.util.history.SubQueryHistory;
import org.apache.tajo.worker.TaskRunnerHistory;
import org.apache.tajo.worker.TaskRunner;
@@ -42,28 +42,28 @@ import static org.apache.tajo.conf.TajoConf.ConfVars;
public class JSPUtil {
static DecimalFormat decimalF = new DecimalFormat("###.0");
- public static void sortQueryUnitArray(QueryUnit[] queryUnits, String sortField, String sortOrder) {
+ public static void sortTaskArray(Task[] tasks, String sortField, String sortOrder) {
if(sortField == null || sortField.isEmpty()) {
sortField = "id";
}
- Arrays.sort(queryUnits, new QueryUnitComparator(sortField, "asc".equals(sortOrder)));
+ Arrays.sort(tasks, new TaskComparator(sortField, "asc".equals(sortOrder)));
}
- public static void sortQueryUnit(List<QueryUnit> queryUnits, String sortField, String sortOrder) {
+ public static void sortTasks(List<Task> tasks, String sortField, String sortOrder) {
if(sortField == null || sortField.isEmpty()) {
sortField = "id";
}
- Collections.sort(queryUnits, new QueryUnitComparator(sortField, "asc".equals(sortOrder)));
+ Collections.sort(tasks, new TaskComparator(sortField, "asc".equals(sortOrder)));
}
- public static void sortQueryUnitHistory(List<QueryUnitHistory> queryUnits, String sortField, String sortOrder) {
+ public static void sortTaskHistory(List<TaskHistory> tasks, String sortField, String sortOrder) {
if(sortField == null || sortField.isEmpty()) {
sortField = "id";
}
- Collections.sort(queryUnits, new QueryUnitHistoryComparator(sortField, "asc".equals(sortOrder)));
+ Collections.sort(tasks, new TaskHistoryComparator(sortField, "asc".equals(sortOrder)));
}
public static void sortTaskRunner(List<TaskRunner> taskRunners) {
@@ -204,95 +204,95 @@ public class JSPUtil {
return activeLabel;
}
- static class QueryUnitComparator implements Comparator<QueryUnit> {
+ static class TaskComparator implements Comparator<Task> {
private String sortField;
private boolean asc;
- public QueryUnitComparator(String sortField, boolean asc) {
+ public TaskComparator(String sortField, boolean asc) {
this.sortField = sortField;
this.asc = asc;
}
@Override
- public int compare(QueryUnit queryUnit, QueryUnit queryUnit2) {
+ public int compare(Task task, Task task2) {
if(asc) {
if("id".equals(sortField)) {
- return queryUnit.getId().compareTo(queryUnit2.getId());
+ return task.getId().compareTo(task2.getId());
} else if("host".equals(sortField)) {
- String host1 = queryUnit.getSucceededHost() == null ? "-" : queryUnit.getSucceededHost();
- String host2 = queryUnit2.getSucceededHost() == null ? "-" : queryUnit2.getSucceededHost();
+ String host1 = task.getSucceededHost() == null ? "-" : task.getSucceededHost();
+ String host2 = task2.getSucceededHost() == null ? "-" : task2.getSucceededHost();
return host1.compareTo(host2);
} else if("runTime".equals(sortField)) {
- return compareLong(queryUnit.getRunningTime(), queryUnit2.getRunningTime());
+ return compareLong(task.getRunningTime(), task2.getRunningTime());
} else if("startTime".equals(sortField)) {
- return compareLong(queryUnit.getLaunchTime(), queryUnit2.getLaunchTime());
+ return compareLong(task.getLaunchTime(), task2.getLaunchTime());
} else {
- return queryUnit.getId().compareTo(queryUnit2.getId());
+ return task.getId().compareTo(task2.getId());
}
} else {
if("id".equals(sortField)) {
- return queryUnit2.getId().compareTo(queryUnit.getId());
+ return task2.getId().compareTo(task.getId());
} else if("host".equals(sortField)) {
- String host1 = queryUnit.getSucceededHost() == null ? "-" : queryUnit.getSucceededHost();
- String host2 = queryUnit2.getSucceededHost() == null ? "-" : queryUnit2.getSucceededHost();
+ String host1 = task.getSucceededHost() == null ? "-" : task.getSucceededHost();
+ String host2 = task2.getSucceededHost() == null ? "-" : task2.getSucceededHost();
return host2.compareTo(host1);
} else if("runTime".equals(sortField)) {
- if(queryUnit2.getLaunchTime() == 0) {
+ if(task2.getLaunchTime() == 0) {
return -1;
- } else if(queryUnit.getLaunchTime() == 0) {
+ } else if(task.getLaunchTime() == 0) {
return 1;
}
- return compareLong(queryUnit2.getRunningTime(), queryUnit.getRunningTime());
+ return compareLong(task2.getRunningTime(), task.getRunningTime());
} else if("startTime".equals(sortField)) {
- return compareLong(queryUnit2.getLaunchTime(), queryUnit.getLaunchTime());
+ return compareLong(task2.getLaunchTime(), task.getLaunchTime());
} else {
- return queryUnit2.getId().compareTo(queryUnit.getId());
+ return task2.getId().compareTo(task.getId());
}
}
}
}
- static class QueryUnitHistoryComparator implements Comparator<QueryUnitHistory> {
+ static class TaskHistoryComparator implements Comparator<TaskHistory> {
private String sortField;
private boolean asc;
- public QueryUnitHistoryComparator(String sortField, boolean asc) {
+ public TaskHistoryComparator(String sortField, boolean asc) {
this.sortField = sortField;
this.asc = asc;
}
@Override
- public int compare(QueryUnitHistory queryUnit, QueryUnitHistory queryUnit2) {
+ public int compare(TaskHistory task1, TaskHistory task2) {
if(asc) {
if("id".equals(sortField)) {
- return queryUnit.getId().compareTo(queryUnit2.getId());
+ return task1.getId().compareTo(task2.getId());
} else if("host".equals(sortField)) {
- String host1 = queryUnit.getHostAndPort() == null ? "-" : queryUnit.getHostAndPort();
- String host2 = queryUnit2.getHostAndPort() == null ? "-" : queryUnit2.getHostAndPort();
+ String host1 = task1.getHostAndPort() == null ? "-" : task1.getHostAndPort();
+ String host2 = task2.getHostAndPort() == null ? "-" : task2.getHostAndPort();
return host1.compareTo(host2);
} else if("runTime".equals(sortField)) {
- return compareLong(queryUnit.getRunningTime(), queryUnit2.getRunningTime());
+ return compareLong(task1.getRunningTime(), task2.getRunningTime());
} else if("startTime".equals(sortField)) {
- return compareLong(queryUnit.getLaunchTime(), queryUnit2.getLaunchTime());
+ return compareLong(task1.getLaunchTime(), task2.getLaunchTime());
} else {
- return queryUnit.getId().compareTo(queryUnit2.getId());
+ return task1.getId().compareTo(task2.getId());
}
} else {
if("id".equals(sortField)) {
- return queryUnit2.getId().compareTo(queryUnit.getId());
+ return task2.getId().compareTo(task1.getId());
} else if("host".equals(sortField)) {
- String host1 = queryUnit.getHostAndPort() == null ? "-" : queryUnit.getHostAndPort();
- String host2 = queryUnit2.getHostAndPort() == null ? "-" : queryUnit2.getHostAndPort();
+ String host1 = task1.getHostAndPort() == null ? "-" : task1.getHostAndPort();
+ String host2 = task2.getHostAndPort() == null ? "-" : task2.getHostAndPort();
return host2.compareTo(host1);
} else if("runTime".equals(sortField)) {
- if(queryUnit2.getLaunchTime() == 0) {
+ if(task2.getLaunchTime() == 0) {
return -1;
- } else if(queryUnit.getLaunchTime() == 0) {
+ } else if(task1.getLaunchTime() == 0) {
return 1;
}
- return compareLong(queryUnit2.getRunningTime(), queryUnit.getRunningTime());
+ return compareLong(task2.getRunningTime(), task1.getRunningTime());
} else if("startTime".equals(sortField)) {
- return compareLong(queryUnit2.getLaunchTime(), queryUnit.getLaunchTime());
+ return compareLong(task2.getLaunchTime(), task1.getLaunchTime());
} else {
- return queryUnit2.getId().compareTo(queryUnit.getId());
+ return task2.getId().compareTo(task1.getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
index 21bc725..9fb427f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java
@@ -24,11 +24,10 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskHistoryProto;
import org.apache.tajo.master.querymaster.QueryInfo;
-import org.apache.tajo.worker.TaskHistory;
import java.io.EOFException;
import java.io.IOException;
@@ -187,22 +186,21 @@ public class HistoryReader {
}
}
- public List<QueryUnitHistory> getQueryUnitHistory(String queryId, String ebId) throws IOException {
+ public List<TaskHistory> getTaskHistory(String queryId, String ebId) throws IOException {
Path queryHistoryFile = getQueryHistoryFilePath(queryId, 0);
if (queryHistoryFile == null) {
- return new ArrayList<QueryUnitHistory>();
+ return new ArrayList<TaskHistory>();
}
Path detailFile = new Path(queryHistoryFile.getParent(), ebId + HistoryWriter.HISTORY_FILE_POSTFIX);
FileSystem fs = HistoryWriter.getNonCrcFileSystem(detailFile, tajoConf);
if (!fs.exists(detailFile)) {
- return new ArrayList<QueryUnitHistory>();
+ return new ArrayList<TaskHistory>();
}
FileStatus fileStatus = fs.getFileStatus(detailFile);
if (fileStatus.getLen() > 100 * 1024 * 1024) { // 100MB
- throw new IOException("QueryUnitHistory file is too big: " +
- detailFile + ", " + fileStatus.getLen() + " bytes");
+ throw new IOException("TaskHistory file is too big: " + detailFile + ", " + fileStatus.getLen() + " bytes");
}
FSDataInputStream in = null;
@@ -212,7 +210,7 @@ public class HistoryReader {
in.readFully(buf, 0, buf.length);
- return SubQueryHistory.fromJsonQueryUnits(new String(buf));
+ return SubQueryHistory.fromJsonTasks(new String(buf));
} finally {
if (in != null) {
in.close();
@@ -220,7 +218,7 @@ public class HistoryReader {
}
}
- public TaskHistory getTaskHistory(String queryUnitAttemptId, long startTime) throws IOException {
+ public org.apache.tajo.worker.TaskHistory getTaskHistory(String taskAttemptId, long startTime) throws IOException {
FileSystem fs = HistoryWriter.getNonCrcFileSystem(taskHistoryParentPath, tajoConf);
SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHH");
@@ -276,9 +274,9 @@ public class HistoryReader {
builder.clear();
TaskHistoryProto taskHistoryProto = builder.mergeFrom(buf).build();
- QueryUnitAttemptId attemptId = new QueryUnitAttemptId(taskHistoryProto.getQueryUnitAttemptId());
- if (attemptId.toString().equals(queryUnitAttemptId)) {
- return new TaskHistory(taskHistoryProto);
+ TaskAttemptId attemptId = new TaskAttemptId(taskHistoryProto.getTaskAttemptId());
+ if (attemptId.toString().equals(taskAttemptId)) {
+ return new org.apache.tajo.worker.TaskHistory(taskHistoryProto);
}
}
} catch (EOFException e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
index 17c9366..7e30f9c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java
@@ -238,7 +238,7 @@ public class HistoryWriter extends AbstractService {
writeTaskHistory((TaskHistory) eachHistory);
} catch (Exception e) {
LOG.error("Error while saving task history: " +
- ((TaskHistory) eachHistory).getQueryUnitAttemptId() + ":" + e.getMessage(), e);
+ ((TaskHistory) eachHistory).getTaskAttemptId() + ":" + e.getMessage(), e);
}
break;
case QUERY:
@@ -301,7 +301,7 @@ public class HistoryWriter extends AbstractService {
out = null;
try {
out = fs.create(path);
- out.write(subQueryHistory.toQueryUnitsJson().getBytes());
+ out.write(subQueryHistory.toTasksJson().getBytes());
LOG.info("Saving query unit: " + path);
} finally {
if (out != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/history/QueryUnitHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryUnitHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryUnitHistory.java
deleted file mode 100644
index 556a971..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryUnitHistory.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util.history;
-
-import com.google.gson.annotations.Expose;
-import org.apache.tajo.engine.json.CoreGsonHelper;
-import org.apache.tajo.json.GsonObject;
-
-public class QueryUnitHistory implements GsonObject {
- @Expose private String id;
- @Expose private String hostAndPort;
- @Expose private int httpPort;
- @Expose private String state;
- @Expose private float progress;
- @Expose private long launchTime;
- @Expose private long finishTime;
- @Expose private int retryCount;
-
- @Expose private int numShuffles;
- @Expose private String shuffleKey;
- @Expose private String shuffleFileName;
-
- @Expose private String[] dataLocations;
- @Expose private String[] fragments;
- @Expose private String[][] fetchs;
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getHostAndPort() {
- return hostAndPort;
- }
-
- public void setHostAndPort(String hostAndPort) {
- this.hostAndPort = hostAndPort;
- }
-
- public int getHttpPort() {
- return httpPort;
- }
-
- public void setHttpPort(int httpPort) {
- this.httpPort = httpPort;
- }
-
- public int getRetryCount() {
- return retryCount;
- }
-
- public void setRetryCount(int retryCount) {
- this.retryCount = retryCount;
- }
-
- public String getState() {
- return state;
- }
-
- public void setState(String state) {
- this.state = state;
- }
-
- public float getProgress() {
- return progress;
- }
-
- public void setProgress(float progress) {
- this.progress = progress;
- }
-
- public long getLaunchTime() {
- return launchTime;
- }
-
- public void setLaunchTime(long launchTime) {
- this.launchTime = launchTime;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- public int getNumShuffles() {
- return numShuffles;
- }
-
- public void setNumShuffles(int numShuffles) {
- this.numShuffles = numShuffles;
- }
-
- public String getShuffleKey() {
- return shuffleKey;
- }
-
- public void setShuffleKey(String shuffleKey) {
- this.shuffleKey = shuffleKey;
- }
-
- public String getShuffleFileName() {
- return shuffleFileName;
- }
-
- public void setShuffleFileName(String shuffleFileName) {
- this.shuffleFileName = shuffleFileName;
- }
-
- public String[] getDataLocations() {
- return dataLocations;
- }
-
- public void setDataLocations(String[] dataLocations) {
- this.dataLocations = dataLocations;
- }
-
- public String[] getFragments() {
- return fragments;
- }
-
- public void setFragments(String[] fragments) {
- this.fragments = fragments;
- }
-
- public String[][] getFetchs() {
- return fetchs;
- }
-
- public void setFetchs(String[][] fetchs) {
- this.fetchs = fetchs;
- }
-
- @Override
- public String toJson() {
- return CoreGsonHelper.toJson(this, QueryUnitHistory.class);
- }
-
- public long getRunningTime() {
- if(finishTime > 0) {
- return finishTime - launchTime;
- } else {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/history/SubQueryHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/SubQueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/SubQueryHistory.java
index b3ac4d2..0afdf5a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/util/history/SubQueryHistory.java
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/SubQueryHistory.java
@@ -67,7 +67,7 @@ public class SubQueryHistory implements GsonObject {
@Expose
private int rackLocalAssigned;
- private List<QueryUnitHistory> queryUnits;
+ private List<TaskHistory> tasks;
public String getExecutionBlockId() {
return executionBlockId;
@@ -213,12 +213,12 @@ public class SubQueryHistory implements GsonObject {
this.killedObjectCount = killedObjectCount;
}
- public List<QueryUnitHistory> getQueryUnits() {
- return queryUnits;
+ public List<TaskHistory> getTasks() {
+ return tasks;
}
- public void setQueryUnits(List<QueryUnitHistory> queryUnits) {
- this.queryUnits = queryUnits;
+ public void setTasks(List<TaskHistory> tasks) {
+ this.tasks = tasks;
}
@Override
@@ -226,19 +226,19 @@ public class SubQueryHistory implements GsonObject {
return CoreGsonHelper.toJson(this, SubQueryHistory.class);
}
- public String toQueryUnitsJson() {
- if (queryUnits == null) {
+ public String toTasksJson() {
+ if (tasks == null) {
return "";
}
- return CoreGsonHelper.getInstance().toJson(queryUnits, new TypeToken<List<QueryUnitHistory>>() {
+ return CoreGsonHelper.getInstance().toJson(tasks, new TypeToken<List<TaskHistory>>() {
}.getType());
}
- public static List<QueryUnitHistory> fromJsonQueryUnits(String json) {
+ public static List<TaskHistory> fromJsonTasks(String json) {
if (json == null || json.trim().isEmpty()) {
- return new ArrayList<QueryUnitHistory>();
+ return new ArrayList<TaskHistory>();
}
- return CoreGsonHelper.getInstance().fromJson(json, new TypeToken<List<QueryUnitHistory>>() {
+ return CoreGsonHelper.getInstance().fromJson(json, new TypeToken<List<TaskHistory>>() {
}.getType());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java
new file mode 100644
index 0000000..35c56e1
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/util/history/TaskHistory.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util.history;
+
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.engine.json.CoreGsonHelper;
+import org.apache.tajo.json.GsonObject;
+
+public class TaskHistory implements GsonObject {
+ @Expose private String id;
+ @Expose private String hostAndPort;
+ @Expose private int httpPort;
+ @Expose private String state;
+ @Expose private float progress;
+ @Expose private long launchTime;
+ @Expose private long finishTime;
+ @Expose private int retryCount;
+
+ @Expose private int numShuffles;
+ @Expose private String shuffleKey;
+ @Expose private String shuffleFileName;
+
+ @Expose private String[] dataLocations;
+ @Expose private String[] fragments;
+ @Expose private String[][] fetchs;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getHostAndPort() {
+ return hostAndPort;
+ }
+
+ public void setHostAndPort(String hostAndPort) {
+ this.hostAndPort = hostAndPort;
+ }
+
+ public int getHttpPort() {
+ return httpPort;
+ }
+
+ public void setHttpPort(int httpPort) {
+ this.httpPort = httpPort;
+ }
+
+ public int getRetryCount() {
+ return retryCount;
+ }
+
+ public void setRetryCount(int retryCount) {
+ this.retryCount = retryCount;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public float getProgress() {
+ return progress;
+ }
+
+ public void setProgress(float progress) {
+ this.progress = progress;
+ }
+
+ public long getLaunchTime() {
+ return launchTime;
+ }
+
+ public void setLaunchTime(long launchTime) {
+ this.launchTime = launchTime;
+ }
+
+ public long getFinishTime() {
+ return finishTime;
+ }
+
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
+
+ public int getNumShuffles() {
+ return numShuffles;
+ }
+
+ public void setNumShuffles(int numShuffles) {
+ this.numShuffles = numShuffles;
+ }
+
+ public String getShuffleKey() {
+ return shuffleKey;
+ }
+
+ public void setShuffleKey(String shuffleKey) {
+ this.shuffleKey = shuffleKey;
+ }
+
+ public String getShuffleFileName() {
+ return shuffleFileName;
+ }
+
+ public void setShuffleFileName(String shuffleFileName) {
+ this.shuffleFileName = shuffleFileName;
+ }
+
+ public String[] getDataLocations() {
+ return dataLocations;
+ }
+
+ public void setDataLocations(String[] dataLocations) {
+ this.dataLocations = dataLocations;
+ }
+
+ public String[] getFragments() {
+ return fragments;
+ }
+
+ public void setFragments(String[] fragments) {
+ this.fragments = fragments;
+ }
+
+ public String[][] getFetchs() {
+ return fetchs;
+ }
+
+ public void setFetchs(String[][] fetchs) {
+ this.fetchs = fetchs;
+ }
+
+ @Override
+ public String toJson() {
+ return CoreGsonHelper.toJson(this, TaskHistory.class);
+ }
+
+ public long getRunningTime() {
+ if(finishTime > 0) {
+ return finishTime - launchTime;
+ } else {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
index f18723f..dd3ee68 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
@@ -92,7 +92,7 @@ public class ExecutionBlockContext {
private AtomicBoolean stop = new AtomicBoolean();
// It keeps all of the query unit attempts while a TaskRunner is running.
- private final ConcurrentMap<QueryUnitAttemptId, Task> tasks = Maps.newConcurrentMap();
+ private final ConcurrentMap<TaskAttemptId, Task> tasks = Maps.newConcurrentMap();
private final ConcurrentMap<String, TaskRunnerHistory> histories = Maps.newConcurrentMap();
@@ -242,12 +242,12 @@ public class ExecutionBlockContext {
return executionBlockId;
}
- public Map<QueryUnitAttemptId, Task> getTasks() {
+ public Map<TaskAttemptId, Task> getTasks() {
return tasks;
}
- public Task getTask(QueryUnitAttemptId queryUnitAttemptId){
- return tasks.get(queryUnitAttemptId);
+ public Task getTask(TaskAttemptId taskAttemptId){
+ return tasks.get(taskAttemptId);
}
public void stopTaskRunner(String id){
@@ -258,7 +258,7 @@ public class ExecutionBlockContext {
return manager.getTaskRunner(taskRunnerId);
}
- public void addTaskHistory(String taskRunnerId, QueryUnitAttemptId quAttemptId, TaskHistory taskHistory) {
+ public void addTaskHistory(String taskRunnerId, TaskAttemptId quAttemptId, TaskHistory taskHistory) {
histories.get(taskRunnerId).addTaskHistory(quAttemptId, taskHistory);
}
[02/13] tajo git commit: TAJO-1221: HA TajoClient should not connect
TajoMaster at the first. (jaehwa)
Posted by ji...@apache.org.
TAJO-1221: HA TajoClient should not connect TajoMaster at the first. (jaehwa)
Closes #286
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/2548768c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/2548768c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/2548768c
Branch: refs/heads/index_support
Commit: 2548768cfed8f018727f6f054317b2adb4d7f485
Parents: e025e3c
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Dec 18 11:41:13 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Dec 18 11:43:06 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../tajo/catalog/AbstractCatalogClient.java | 2 +-
.../org/apache/tajo/cli/tools/TajoAdmin.java | 2 +-
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 2 +-
.../apache/tajo/client/SessionConnection.java | 2 +-
.../org/apache/tajo/client/TajoClientImpl.java | 5 +-
.../apache/tajo/client/TajoHAClientUtil.java | 20 +-
.../java/org/apache/tajo/ha/HAConstants.java | 27 ++
.../java/org/apache/tajo/ha/HAServiceUtil.java | 292 ++++++++++++++++++
.../org/apache/tajo/util/HAServiceUtil.java | 293 -------------------
.../apache/tajo/master/TajoContainerProxy.java | 2 +-
.../tajo/master/ha/HAServiceHDFSImpl.java | 100 +++++--
.../master/querymaster/QueryJobManager.java | 5 +-
.../tajo/master/querymaster/QueryMaster.java | 2 +-
.../master/querymaster/QueryMasterTask.java | 2 +-
.../tajo/worker/TajoResourceAllocator.java | 2 +-
.../java/org/apache/tajo/worker/TajoWorker.java | 1 +
.../tajo/worker/WorkerHeartbeatService.java | 2 +-
.../ConnectivityCheckerRuleForTajoWorker.java | 2 +-
.../org/apache/tajo/TajoTestingCluster.java | 3 +
.../tajo/master/ha/TestHAServiceHDFSImpl.java | 115 +++++---
21 files changed, 494 insertions(+), 389 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 65492d2..de63285 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1221: HA TajoClient should not connect TajoMaster at the first. (jaehwa)
+
TAJO-1241: Change default client and table time zone behavior. (hyunsik)
TAJO-1243: *-site.xml.template should have default configs commented out.
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index dde6980..6b50115 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -33,7 +33,7 @@ import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.rpc.ServerCallable;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.ProtoUtil;
import java.net.InetSocketAddress;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java b/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
index 5ef8d76..88b8e0f 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java
@@ -28,7 +28,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.ipc.ClientProtos.BriefQueryInfo;
import org.apache.tajo.ipc.ClientProtos.WorkerResourceInfo;
import org.apache.tajo.util.NetUtils;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.TajoIdUtils;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java b/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java
index 12d94ad..ef8fee9 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tools/TajoHAAdmin.java
@@ -24,7 +24,7 @@ import org.apache.tajo.client.TajoClient;
import org.apache.tajo.client.TajoClientImpl;
import org.apache.tajo.client.TajoHAClientUtil;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import java.io.IOException;
import java.io.PrintWriter;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
index 44b772b..db2bd2a 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
@@ -33,7 +33,7 @@ import org.apache.tajo.ipc.TajoMasterClientProtocol;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.rpc.ServerCallable;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.ProtoUtil;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
index dff8d65..8eafc91 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
@@ -33,6 +33,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.ipc.ClientProtos.*;
import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.jdbc.TajoResultSet;
@@ -56,11 +57,11 @@ public class TajoClientImpl extends SessionConnection implements TajoClient, Que
CatalogAdminClient catalogClient;
public TajoClientImpl(TajoConf conf) throws IOException {
- this(conf, NetUtils.createSocketAddr(conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS)), null);
+ this(conf, TajoHAClientUtil.getRpcClientAddress(conf), null);
}
public TajoClientImpl(TajoConf conf, @Nullable String baseDatabase) throws IOException {
- this(conf, NetUtils.createSocketAddr(conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS)), baseDatabase);
+ this(conf, TajoHAClientUtil.getRpcClientAddress(conf), baseDatabase);
}
public TajoClientImpl(InetSocketAddress addr) throws IOException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-client/src/main/java/org/apache/tajo/client/TajoHAClientUtil.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoHAClientUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoHAClientUtil.java
index b95fb35..12a9ec8 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoHAClientUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoHAClientUtil.java
@@ -39,12 +39,13 @@ package org.apache.tajo.client;
import com.google.protobuf.ServiceException;
import org.apache.tajo.cli.tsql.TajoCli.TajoCliContext;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
+import org.apache.tajo.util.NetUtils;
import java.io.IOException;
+import java.net.InetSocketAddress;
public class TajoHAClientUtil {
-
/**
* In TajoMaster HA mode, if TajoCli can't connect existing active master,
* this should try to connect new active master.
@@ -65,11 +66,11 @@ public class TajoHAClientUtil {
if (conf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) {
if (!HAServiceUtil.isMasterAlive(conf.getVar(
- TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS), conf)) {
+ TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS), conf)) {
TajoClient tajoClient = null;
String baseDatabase = client.getBaseDatabase();
conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS,
- HAServiceUtil.getMasterClientName(conf));
+ HAServiceUtil.getMasterClientName(conf));
client.close();
tajoClient = new TajoClientImpl(conf, baseDatabase);
@@ -84,4 +85,15 @@ public class TajoHAClientUtil {
return client;
}
}
+
+
+ public static InetSocketAddress getRpcClientAddress(TajoConf conf) {
+ if (conf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) {
+ return NetUtils.createSocketAddr(HAServiceUtil.getMasterClientName(conf));
+ } else {
+ return NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .TAJO_MASTER_CLIENT_RPC_ADDRESS));
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-common/src/main/java/org/apache/tajo/ha/HAConstants.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/ha/HAConstants.java b/tajo-common/src/main/java/org/apache/tajo/ha/HAConstants.java
new file mode 100644
index 0000000..c5f4b8a
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/ha/HAConstants.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.ha;
+
+public class HAConstants {
+ public final static int MASTER_UMBILICAL_RPC_ADDRESS = 1;
+ public final static int MASTER_CLIENT_RPC_ADDRESS = 2;
+ public final static int RESOURCE_TRACKER_RPC_ADDRESS = 3;
+ public final static int CATALOG_ADDRESS = 4;
+ public final static int MASTER_INFO_ADDRESS = 5;
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-common/src/main/java/org/apache/tajo/ha/HAServiceUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/ha/HAServiceUtil.java b/tajo-common/src/main/java/org/apache/tajo/ha/HAServiceUtil.java
new file mode 100644
index 0000000..b62d73b
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/ha/HAServiceUtil.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.ha;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.*;
+import org.apache.tajo.TajoConstants;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.util.NetUtils;
+
+
+import javax.net.SocketFactory;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HAServiceUtil {
+ private static Log LOG = LogFactory.getLog(HAServiceUtil.class);
+
+ public static InetSocketAddress getMasterUmbilicalAddress(TajoConf conf) {
+ return getMasterAddress(conf, HAConstants.MASTER_UMBILICAL_RPC_ADDRESS);
+ }
+
+ public static String getMasterUmbilicalName(TajoConf conf) {
+ return NetUtils.normalizeInetSocketAddress(getMasterUmbilicalAddress(conf));
+ }
+
+ public static InetSocketAddress getMasterClientAddress(TajoConf conf) {
+ return getMasterAddress(conf, HAConstants.MASTER_CLIENT_RPC_ADDRESS);
+ }
+
+ public static String getMasterClientName(TajoConf conf) {
+ return NetUtils.normalizeInetSocketAddress(getMasterClientAddress(conf));
+ }
+
+ public static InetSocketAddress getResourceTrackerAddress(TajoConf conf) {
+ return getMasterAddress(conf, HAConstants.RESOURCE_TRACKER_RPC_ADDRESS);
+ }
+
+ public static String getResourceTrackerName(TajoConf conf) {
+ return NetUtils.normalizeInetSocketAddress(getResourceTrackerAddress(conf));
+ }
+
+ public static InetSocketAddress getCatalogAddress(TajoConf conf) {
+ return getMasterAddress(conf, HAConstants.CATALOG_ADDRESS);
+ }
+
+ public static String getCatalogName(TajoConf conf) {
+ return NetUtils.normalizeInetSocketAddress(getCatalogAddress(conf));
+ }
+
+ public static InetSocketAddress getMasterInfoAddress(TajoConf conf) {
+ return getMasterAddress(conf, HAConstants.MASTER_INFO_ADDRESS);
+ }
+
+ public static String getMasterInfoName(TajoConf conf) {
+ return NetUtils.normalizeInetSocketAddress(getMasterInfoAddress(conf));
+ }
+
+ public static InetSocketAddress getMasterAddress(TajoConf conf, int type) {
+ InetSocketAddress masterAddress = null;
+
+ if (conf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) {
+ try {
+ FileSystem fs = getFileSystem(conf);
+ Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
+
+ if (fs.exists(activePath)) {
+ FileStatus[] files = fs.listStatus(activePath);
+
+ if (files.length == 1) {
+ Path file = files[0].getPath();
+ String hostAddress = file.getName().replaceAll("_", ":");
+ FSDataInputStream stream = fs.open(file);
+ String data = stream.readUTF();
+ stream.close();
+
+ String[] addresses = data.split("_");
+
+ switch (type) {
+ case 1:
+ masterAddress = NetUtils.createSocketAddr(hostAddress);
+ break;
+ case 2:
+ masterAddress = NetUtils.createSocketAddr(addresses[0]);
+ break;
+ case 3:
+ masterAddress = NetUtils.createSocketAddr(addresses[1]);
+ break;
+ case 4:
+ masterAddress = NetUtils.createSocketAddr(addresses[2]);
+ break;
+ case 5:
+ masterAddress = NetUtils.createSocketAddr(addresses[3]);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ if (masterAddress == null) {
+ switch (type) {
+ case 1:
+ masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .TAJO_MASTER_UMBILICAL_RPC_ADDRESS));
+ break;
+ case 2:
+ masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .TAJO_MASTER_CLIENT_RPC_ADDRESS));
+ break;
+ case 3:
+ masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .RESOURCE_TRACKER_RPC_ADDRESS));
+ break;
+ case 4:
+ masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .CATALOG_ADDRESS));
+ break;
+ case 5:
+ masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
+ .TAJO_MASTER_INFO_ADDRESS));
+ break;
+ default:
+ break;
+ }
+ }
+
+ return masterAddress;
+ }
+
+ public static boolean isMasterAlive(InetSocketAddress masterAddress, TajoConf conf) {
+ return isMasterAlive(NetUtils.normalizeInetSocketAddress(masterAddress), conf);
+ }
+
+ public static boolean isMasterAlive(String masterName, TajoConf conf) {
+ boolean isAlive = true;
+
+ try {
+ // how to create sockets
+ SocketFactory socketFactory = org.apache.hadoop.net.NetUtils.getDefaultSocketFactory(conf);
+
+ int connectionTimeout = conf.getInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT);
+
+ InetSocketAddress server = org.apache.hadoop.net.NetUtils.createSocketAddr(masterName);
+
+ // connected socket
+ Socket socket = socketFactory.createSocket();
+ org.apache.hadoop.net.NetUtils.connect(socket, server, connectionTimeout);
+ } catch (Exception e) {
+ isAlive = false;
+ }
+ return isAlive;
+ }
+
+ public static int getState(String masterName, TajoConf conf) {
+ String targetMaster = masterName.replaceAll(":", "_");
+ int retValue = -1;
+
+ try {
+ FileSystem fs = getFileSystem(conf);
+ Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
+ Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
+
+ Path temPath = null;
+
+ // Check backup masters
+ FileStatus[] files = fs.listStatus(backupPath);
+ for (FileStatus status : files) {
+ temPath = status.getPath();
+ if (temPath.getName().equals(targetMaster)) {
+ return 0;
+ }
+ }
+
+ // Check active master
+ files = fs.listStatus(activePath);
+ if (files.length == 1) {
+ temPath = files[0].getPath();
+ if (temPath.getName().equals(targetMaster)) {
+ return 1;
+ }
+ }
+ retValue = -2;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return retValue;
+ }
+
+ public static int formatHA(TajoConf conf) {
+ int retValue = -1;
+ try {
+ FileSystem fs = getFileSystem(conf);
+ Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
+ Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
+ Path temPath = null;
+
+ int aliveMasterCount = 0;
+ // Check backup masters
+ FileStatus[] files = fs.listStatus(backupPath);
+ for (FileStatus status : files) {
+ temPath = status.getPath();
+ if (isMasterAlive(temPath.getName().replaceAll("_", ":"), conf)) {
+ aliveMasterCount++;
+ }
+ }
+
+ // Check active master
+ files = fs.listStatus(activePath);
+ if (files.length == 1) {
+ temPath = files[0].getPath();
+ if (isMasterAlive(temPath.getName().replaceAll("_", ":"), conf)) {
+ aliveMasterCount++;
+ }
+ }
+
+ // If there is any alive master, users can't format storage.
+ if (aliveMasterCount > 0) {
+ return 0;
+ }
+
+ // delete ha path.
+ fs.delete(TajoConf.getSystemHADir(conf), true);
+ retValue = 1;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return retValue;
+ }
+
+
+ public static List<String> getMasters(TajoConf conf) {
+ List<String> list = new ArrayList<String>();
+
+ try {
+ FileSystem fs = getFileSystem(conf);
+ Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
+ Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
+ Path temPath = null;
+
+ // Check backup masters
+ FileStatus[] files = fs.listStatus(backupPath);
+ for (FileStatus status : files) {
+ temPath = status.getPath();
+ list.add(temPath.getName().replaceAll("_", ":"));
+ }
+
+ // Check active master
+ files = fs.listStatus(activePath);
+ if (files.length == 1) {
+ temPath = files[0].getPath();
+ list.add(temPath.getName().replaceAll("_", ":"));
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return list;
+ }
+
+ private static FileSystem getFileSystem(TajoConf conf) throws IOException {
+ Path rootPath = TajoConf.getTajoRootDir(conf);
+ return rootPath.getFileSystem(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-common/src/main/java/org/apache/tajo/util/HAServiceUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/HAServiceUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/HAServiceUtil.java
deleted file mode 100644
index 4f03113..0000000
--- a/tajo-common/src/main/java/org/apache/tajo/util/HAServiceUtil.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import org.apache.hadoop.fs.*;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.conf.TajoConf;
-
-import javax.net.SocketFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.List;
-
-public class HAServiceUtil {
-
- private final static int MASTER_UMBILICAL_RPC_ADDRESS = 1;
- private final static int MASTER_CLIENT_RPC_ADDRESS = 2;
- private final static int RESOURCE_TRACKER_RPC_ADDRESS = 3;
- private final static int CATALOG_ADDRESS = 4;
- private final static int MASTER_INFO_ADDRESS = 5;
-
- public static InetSocketAddress getMasterUmbilicalAddress(TajoConf conf) {
- return getMasterAddress(conf, MASTER_UMBILICAL_RPC_ADDRESS);
- }
-
- public static String getMasterUmbilicalName(TajoConf conf) {
- return NetUtils.normalizeInetSocketAddress(getMasterUmbilicalAddress(conf));
- }
-
- public static InetSocketAddress getMasterClientAddress(TajoConf conf) {
- return getMasterAddress(conf, MASTER_CLIENT_RPC_ADDRESS);
- }
-
- public static String getMasterClientName(TajoConf conf) {
- return NetUtils.normalizeInetSocketAddress(getMasterClientAddress(conf));
- }
-
- public static InetSocketAddress getResourceTrackerAddress(TajoConf conf) {
- return getMasterAddress(conf, RESOURCE_TRACKER_RPC_ADDRESS);
- }
-
- public static String getResourceTrackerName(TajoConf conf) {
- return NetUtils.normalizeInetSocketAddress(getResourceTrackerAddress(conf));
- }
-
- public static InetSocketAddress getCatalogAddress(TajoConf conf) {
- return getMasterAddress(conf, CATALOG_ADDRESS);
- }
-
- public static String getCatalogName(TajoConf conf) {
- return NetUtils.normalizeInetSocketAddress(getCatalogAddress(conf));
- }
-
- public static InetSocketAddress getMasterInfoAddress(TajoConf conf) {
- return getMasterAddress(conf, MASTER_INFO_ADDRESS);
- }
-
- public static String getMasterInfoName(TajoConf conf) {
- return NetUtils.normalizeInetSocketAddress(getMasterInfoAddress(conf));
- }
-
- public static InetSocketAddress getMasterAddress(TajoConf conf, int type) {
- InetSocketAddress masterAddress = null;
-
- if (conf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) {
- try {
- FileSystem fs = getFileSystem(conf);
- Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
-
- if (fs.exists(activePath)) {
- FileStatus[] files = fs.listStatus(activePath);
-
- if (files.length == 1) {
- Path file = files[0].getPath();
- String hostAddress = file.getName().replaceAll("_", ":");
- FSDataInputStream stream = fs.open(file);
- String data = stream.readUTF();
- stream.close();
-
- String[] addresses = data.split("_");
-
- switch (type) {
- case 1:
- masterAddress = NetUtils.createSocketAddr(hostAddress);
- break;
- case 2:
- masterAddress = NetUtils.createSocketAddr(addresses[0]);
- break;
- case 3:
- masterAddress = NetUtils.createSocketAddr(addresses[1]);
- break;
- case 4:
- masterAddress = NetUtils.createSocketAddr(addresses[2]);
- break;
- case 5:
- masterAddress = NetUtils.createSocketAddr(addresses[3]);
- break;
- default:
- break;
- }
- }
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- if (masterAddress == null) {
- switch (type) {
- case 1:
- masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
- .TAJO_MASTER_UMBILICAL_RPC_ADDRESS));
- break;
- case 2:
- masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
- .TAJO_MASTER_CLIENT_RPC_ADDRESS));
- break;
- case 3:
- masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
- .RESOURCE_TRACKER_RPC_ADDRESS));
- break;
- case 4:
- masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
- .CATALOG_ADDRESS));
- break;
- case 5:
- masterAddress = NetUtils.createSocketAddr(conf.getVar(TajoConf.ConfVars
- .TAJO_MASTER_INFO_ADDRESS));
- break;
- default:
- break;
- }
- }
-
- return masterAddress;
- }
-
- public static boolean isMasterAlive(InetSocketAddress masterAddress, TajoConf conf) {
- return isMasterAlive(NetUtils.normalizeInetSocketAddress(masterAddress), conf);
- }
-
- public static boolean isMasterAlive(String masterName, TajoConf conf) {
- boolean isAlive = true;
-
- try {
- // how to create sockets
- SocketFactory socketFactory = org.apache.hadoop.net.NetUtils.getDefaultSocketFactory(conf);
-
- int connectionTimeout = conf.getInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
- CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT);
-
- InetSocketAddress server = org.apache.hadoop.net.NetUtils.createSocketAddr(masterName);
-
- // connected socket
- Socket socket = socketFactory.createSocket();
- org.apache.hadoop.net.NetUtils.connect(socket, server, connectionTimeout);
- } catch (Exception e) {
- isAlive = false;
- }
- return isAlive;
- }
-
- public static int getState(String masterName, TajoConf conf) {
- String targetMaster = masterName.replaceAll(":", "_");
- int retValue = -1;
-
- try {
- FileSystem fs = getFileSystem(conf);
- Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
- Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
-
- Path temPath = null;
-
- // Check backup masters
- FileStatus[] files = fs.listStatus(backupPath);
- for (FileStatus status : files) {
- temPath = status.getPath();
- if (temPath.getName().equals(targetMaster)) {
- return 0;
- }
- }
-
- // Check active master
- files = fs.listStatus(activePath);
- if (files.length == 1) {
- temPath = files[0].getPath();
- if (temPath.getName().equals(targetMaster)) {
- return 1;
- }
- }
- retValue = -2;
- } catch (Exception e) {
- e.printStackTrace();
- }
- return retValue;
- }
-
- public static int formatHA(TajoConf conf) {
- int retValue = -1;
- try {
- FileSystem fs = getFileSystem(conf);
- Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
- Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
- Path temPath = null;
-
- int aliveMasterCount = 0;
- // Check backup masters
- FileStatus[] files = fs.listStatus(backupPath);
- for (FileStatus status : files) {
- temPath = status.getPath();
- if (isMasterAlive(temPath.getName().replaceAll("_", ":"), conf)) {
- aliveMasterCount++;
- }
- }
-
- // Check active master
- files = fs.listStatus(activePath);
- if (files.length == 1) {
- temPath = files[0].getPath();
- if (isMasterAlive(temPath.getName().replaceAll("_", ":"), conf)) {
- aliveMasterCount++;
- }
- }
-
- // If there is any alive master, users can't format storage.
- if (aliveMasterCount > 0) {
- return 0;
- }
-
- // delete ha path.
- fs.delete(TajoConf.getSystemHADir(conf), true);
- retValue = 1;
- } catch (Exception e) {
- e.printStackTrace();
- }
- return retValue;
- }
-
-
- public static List<String> getMasters(TajoConf conf) {
- List<String> list = new ArrayList<String>();
-
- try {
- FileSystem fs = getFileSystem(conf);
- Path activePath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_ACTIVE_DIR_NAME);
- Path backupPath = new Path(TajoConf.getSystemHADir(conf), TajoConstants.SYSTEM_HA_BACKUP_DIR_NAME);
- Path temPath = null;
-
- // Check backup masters
- FileStatus[] files = fs.listStatus(backupPath);
- for (FileStatus status : files) {
- temPath = status.getPath();
- list.add(temPath.getName().replaceAll("_", ":"));
- }
-
- // Check active master
- files = fs.listStatus(activePath);
- if (files.length == 1) {
- temPath = files[0].getPath();
- list.add(temPath.getName().replaceAll("_", ":"));
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- }
- return list;
- }
-
- private static FileSystem getFileSystem(TajoConf conf) throws IOException {
- Path rootPath = TajoConf.getTajoRootDir(conf);
- return rootPath.getFileSystem(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
index 158316e..0d2acf7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
@@ -36,7 +36,7 @@ import org.apache.tajo.master.rm.TajoWorkerContainerId;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
import org.apache.tajo.rpc.RpcConnectionPool;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import java.net.InetSocketAddress;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/master/ha/HAServiceHDFSImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/ha/HAServiceHDFSImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/ha/HAServiceHDFSImpl.java
index 26bc97b..45219b3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/ha/HAServiceHDFSImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/ha/HAServiceHDFSImpl.java
@@ -25,12 +25,14 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.net.NetUtils;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ha.HAConstants;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.TajoMaster.MasterContext;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.TUtil;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.List;
/**
@@ -65,7 +67,10 @@ public class HAServiceHDFSImpl implements HAService {
this.context = context;
this.conf = context.getConf();
initSystemDirectory();
- this.masterName = conf.get(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS.varname);
+
+ InetSocketAddress socketAddress = context.getTajoMasterService().getBindAddress();
+ this.masterName = socketAddress.getAddress().getHostAddress() + ":" + socketAddress.getPort();
+
monitorInterval = conf.getIntVar(TajoConf.ConfVars.TAJO_MASTER_HA_MONITOR_INTERVAL);
}
@@ -133,7 +138,6 @@ public class HAServiceHDFSImpl implements HAService {
}
private void createMasterFile(boolean isActive) throws IOException {
- String hostName = masterName.split(":")[0];
String fileName = masterName.replaceAll(":", "_");
Path path = null;
@@ -144,20 +148,27 @@ public class HAServiceHDFSImpl implements HAService {
}
StringBuilder sb = new StringBuilder();
- sb.append(context.getConf().get(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS.varname,
- hostName + ":26002"));
- sb.append("_");
- sb.append(context.getConf().get(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS.varname,
- hostName + ":26003"));
- sb.append("_");
- sb.append(context.getConf().get(TajoConf.ConfVars.CATALOG_ADDRESS.varname, hostName + ":26005"));
- sb.append("_");
- sb.append(context.getConf().get(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS.varname,
- hostName + ":26080"));
+ InetSocketAddress address = getHostAddress(HAConstants.MASTER_CLIENT_RPC_ADDRESS);
+ sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");
+
+ address = getHostAddress(HAConstants.RESOURCE_TRACKER_RPC_ADDRESS);
+ sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");
+
+ address = getHostAddress(HAConstants.CATALOG_ADDRESS);
+ sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort()).append("_");
+
+ address = getHostAddress(HAConstants.MASTER_INFO_ADDRESS);
+ sb.append(address.getAddress().getHostAddress()).append(":").append(address.getPort());
FSDataOutputStream out = fs.create(path);
- out.writeUTF(sb.toString());
- out.close();
+
+ try {
+ out.writeUTF(sb.toString());
+ out.hflush();
+ out.close();
+ } catch (FileAlreadyExistsException e) {
+ createMasterFile(false);
+ }
if (isActive) {
isActiveStatus = true;
@@ -169,6 +180,36 @@ public class HAServiceHDFSImpl implements HAService {
}
+ private InetSocketAddress getHostAddress(int type) {
+ InetSocketAddress address = null;
+
+ switch (type) {
+ case HAConstants.MASTER_UMBILICAL_RPC_ADDRESS:
+ address = context.getConf().getSocketAddrVar(TajoConf.ConfVars
+ .TAJO_MASTER_UMBILICAL_RPC_ADDRESS);
+ break;
+ case HAConstants.MASTER_CLIENT_RPC_ADDRESS:
+ address = context.getConf().getSocketAddrVar(TajoConf.ConfVars
+ .TAJO_MASTER_CLIENT_RPC_ADDRESS);
+ break;
+ case HAConstants.RESOURCE_TRACKER_RPC_ADDRESS:
+ address = context.getConf().getSocketAddrVar(TajoConf.ConfVars
+ .RESOURCE_TRACKER_RPC_ADDRESS);
+ break;
+ case HAConstants.CATALOG_ADDRESS:
+ address = context.getConf().getSocketAddrVar(TajoConf.ConfVars
+ .CATALOG_ADDRESS);
+ break;
+ case HAConstants.MASTER_INFO_ADDRESS:
+ address = context.getConf().getSocketAddrVar(TajoConf.ConfVars
+ .TAJO_MASTER_INFO_ADDRESS);
+ default:
+ break;
+ }
+
+ return NetUtils.createSocketAddr(masterName.split(":")[0] + ":" + address.getPort());
+ }
+
@Override
public void delete() throws IOException {
String fileName = masterName.replaceAll(":", "_");
@@ -196,9 +237,6 @@ public class HAServiceHDFSImpl implements HAService {
@Override
public List<TajoMasterInfo> getMasters() throws IOException {
List<TajoMasterInfo> list = TUtil.newList();
- boolean isAlive = false;
- TajoMasterInfo info = null;
- String hostAddress = null;
Path path = null;
FileStatus[] files = fs.listStatus(activePath);
@@ -246,20 +284,22 @@ public class HAServiceHDFSImpl implements HAService {
while (!stopped && !Thread.currentThread().isInterrupted()) {
synchronized (HAServiceHDFSImpl.this) {
try {
- boolean isAlive = HAServiceUtil.isMasterAlive(currentActiveMaster, conf);
- if (LOG.isDebugEnabled()) {
- LOG.debug("currentActiveMaster:" + currentActiveMaster + ", thisMasterName:" + masterName
+ if (!currentActiveMaster.equals(masterName)) {
+ boolean isAlive = HAServiceUtil.isMasterAlive(currentActiveMaster, conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("currentActiveMaster:" + currentActiveMaster + ", thisMasterName:" + masterName
+ ", isAlive:" + isAlive);
- }
+ }
- // If active master is dead, this master should be active master instead of
- // previous active master.
- if (!isAlive) {
- FileStatus[] files = fs.listStatus(activePath);
- if (files.length == 0 || (files.length == 1
- && currentActiveMaster.equals(files[0].getPath().getName().replaceAll("_", ":")))) {
- delete();
- register();
+ // If active master is dead, this master should be active master instead of
+ // previous active master.
+ if (!isAlive) {
+ FileStatus[] files = fs.listStatus(activePath);
+ if (files.length == 0 || (files.length == 1
+ && currentActiveMaster.equals(files[0].getPath().getName().replaceAll("_", ":")))) {
+ delete();
+ register();
+ }
}
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
index 536f6ac..ddbd3e1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
@@ -19,6 +19,7 @@
package org.apache.tajo.master.querymaster;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -52,9 +53,9 @@ public class QueryJobManager extends CompositeService {
private SimpleFifoScheduler scheduler;
- private final Map<QueryId, QueryInProgress> submittedQueries = new HashMap<QueryId, QueryInProgress>();
+ private final Map<QueryId, QueryInProgress> submittedQueries = Maps.newConcurrentMap();
- private final Map<QueryId, QueryInProgress> runningQueries = new HashMap<QueryId, QueryInProgress>();
+ private final Map<QueryId, QueryInProgress> runningQueries = Maps.newConcurrentMap();
private AtomicLong minExecutionTime = new AtomicLong(Long.MAX_VALUE);
private AtomicLong maxExecutionTime = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
index 42fac3a..7ddd787 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
@@ -32,6 +32,7 @@ import org.apache.tajo.*;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.TajoAsyncDispatcher;
@@ -41,7 +42,6 @@ import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
-import org.apache.tajo.util.HAServiceUtil;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.history.QueryHistory;
import org.apache.tajo.worker.TajoWorker;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 1eaef0f..8f63416 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -64,7 +64,7 @@ import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.StorageProperty;
import org.apache.tajo.storage.StorageUtil;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.metrics.TajoMetrics;
import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter;
import org.apache.tajo.worker.AbstractResourceAllocator;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
index 9345885..f055733 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
@@ -47,7 +47,7 @@ import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.util.ApplicationIdUtils;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import java.net.InetSocketAddress;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
index 0f0c1f9..4d96529 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
@@ -35,6 +35,7 @@ import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.CatalogClient;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.ha.TajoMasterInfo;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
index f12e83c..c809921 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
@@ -35,7 +35,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.storage.DiskDeviceInfo;
import org.apache.tajo.storage.DiskMountInfo;
import org.apache.tajo.storage.DiskUtil;
-import org.apache.tajo.util.HAServiceUtil;
+import org.apache.tajo.ha.HAServiceUtil;
import java.io.File;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
index 328a31b..6eb710a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
@@ -22,6 +22,7 @@ import java.net.InetSocketAddress;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
@@ -31,7 +32,6 @@ import org.apache.tajo.rule.SelfDiagnosisRuleDefinition;
import org.apache.tajo.rule.SelfDiagnosisRuleVisibility;
import org.apache.tajo.rule.EvaluationResult.EvaluationResultCode;
import org.apache.tajo.rule.SelfDiagnosisRule;
-import org.apache.tajo.util.HAServiceUtil;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.worker.TajoWorker;
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 64c27e0..9868297 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -157,7 +157,10 @@ public class TajoTestingCluster {
if (!StringUtils.isEmpty(LOG_LEVEL)) {
Level defaultLevel = Logger.getRootLogger().getLevel();
Logger.getLogger("org.apache.tajo").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("org.apache.tajo.master.TajoAsyncDispatcher").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(),
+ defaultLevel));
Logger.getLogger("org.apache.hadoop").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
Logger.getLogger("BlockStateChange").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/2548768c/tajo-core/src/test/java/org/apache/tajo/master/ha/TestHAServiceHDFSImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/ha/TestHAServiceHDFSImpl.java b/tajo-core/src/test/java/org/apache/tajo/master/ha/TestHAServiceHDFSImpl.java
index 69ed556..e1806e1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/ha/TestHAServiceHDFSImpl.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/ha/TestHAServiceHDFSImpl.java
@@ -26,12 +26,16 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.client.TajoClientImpl;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.master.TajoMaster;
import org.junit.Test;
+import java.util.List;
+
import static junit.framework.TestCase.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -41,86 +45,90 @@ public class TestHAServiceHDFSImpl {
private static Log LOG = LogFactory.getLog(TestHAServiceHDFSImpl.class);
private TajoTestingCluster cluster;
- private TajoMaster backupMaster1, backupMaster2;
+ private TajoMaster backupMaster;
private TajoConf conf;
private TajoClient client;
- private Path testDir;
private Path haPath, activePath, backupPath;
- private static final String LOCAL_HOST = "localhost:";
+ private String masterAddress;
@Test
- public final void testTwoBackupMasters() throws Exception {
+ public final void testAutoFailOver() throws Exception {
cluster = new TajoTestingCluster(true);
- cluster.startMiniCluster(1);
+ cluster.startMiniCluster(1);
conf = cluster.getConfiguration();
client = new TajoClientImpl(conf);
+
try {
FileSystem fs = cluster.getDefaultFileSystem();
- startBackupMasters();
- verifyMasterAddress();
- verifySystemDirectories(fs);
+ masterAddress = HAServiceUtil.getMasterUmbilicalName(conf).split(":")[0];
+
+ setConfiguration();
- Path backupMasterFile1 = new Path(backupPath, backupMaster1.getMasterName()
- .replaceAll(":", "_"));
- assertTrue(fs.exists(backupMasterFile1));
+ backupMaster = new TajoMaster();
+ backupMaster.init(conf);
+ backupMaster.start();
- Path backupMasterFile2 = new Path(backupPath, backupMaster2.getMasterName()
- .replaceAll(":", "_"));
- assertTrue(fs.exists(backupMasterFile2));
+ assertNotEquals(cluster.getMaster().getMasterName(), backupMaster.getMasterName());
+
+ verifySystemDirectories(fs);
+
+ Path backupMasterFile = new Path(backupPath, backupMaster.getMasterName()
+ .replaceAll(":", "_"));
+ assertTrue(fs.exists(backupMasterFile));
assertTrue(cluster.getMaster().isActiveMaster());
- assertFalse(backupMaster1.isActiveMaster());
- assertFalse(backupMaster2.isActiveMaster());
+ assertFalse(backupMaster.isActiveMaster());
+
+ createDatabaseAndTable();
+ verifyDataBaseAndTable();
+ client.close();
+
+ cluster.getMaster().stop();
+
+ Thread.sleep(7000);
+
+ assertFalse(cluster.getMaster().isActiveMaster());
+ assertTrue(backupMaster.isActiveMaster());
+
+ client = new TajoClientImpl(conf);
+ verifyDataBaseAndTable();
} finally {
- IOUtils.cleanup(LOG, client, backupMaster1, backupMaster2);
+ client.close();
+ backupMaster.stop();
cluster.shutdownMiniCluster();
}
}
- private void startBackupMasters() throws Exception {
-
+ private void setConfiguration() {
conf = cluster.getConfiguration();
- conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
- conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
- conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
- conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
- conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true);
-
- backupMaster1 = new TajoMaster();
- backupMaster1.init(conf);
- backupMaster1.start();
conf.setVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
+ masterAddress + ":" + NetUtils.getFreeSocketPort());
conf.setVar(TajoConf.ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
+ masterAddress + ":" + NetUtils.getFreeSocketPort());
conf.setVar(TajoConf.ConfVars.RESOURCE_TRACKER_RPC_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
+ masterAddress + ":" + NetUtils.getFreeSocketPort());
conf.setVar(TajoConf.ConfVars.CATALOG_ADDRESS,
- LOCAL_HOST + NetUtils.getFreeSocketPort());
+ masterAddress + ":" + NetUtils.getFreeSocketPort());
+ conf.setVar(TajoConf.ConfVars.TAJO_MASTER_INFO_ADDRESS,
+ masterAddress + ":" + NetUtils.getFreeSocketPort());
conf.setBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE, true);
- backupMaster2 = new TajoMaster();
- backupMaster2.init(conf);
- backupMaster2.start();
- }
+ //Client API service RPC Server
+ conf.setIntVar(TajoConf.ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
- private void verifyMasterAddress() {
- assertNotEquals(cluster.getMaster().getMasterName(),
- backupMaster1.getMasterName());
- assertNotEquals(cluster.getMaster().getMasterName(),
- backupMaster2.getMasterName());
- assertNotEquals(backupMaster1.getMasterName(),
- backupMaster2.getMasterName());
+ // Internal RPC Server
+ conf.setIntVar(TajoConf.ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(TajoConf.ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(TajoConf.ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(TajoConf.ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(TajoConf.ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
}
private void verifySystemDirectories(FileSystem fs) throws Exception {
@@ -134,6 +142,17 @@ public class TestHAServiceHDFSImpl {
assertTrue(fs.exists(backupPath));
assertEquals(1, fs.listStatus(activePath).length);
- assertEquals(2, fs.listStatus(backupPath).length);
+ assertEquals(1, fs.listStatus(backupPath).length);
+ }
+
+ private void createDatabaseAndTable() throws Exception {
+ client.executeQuery("CREATE TABLE default.table1 (age int);");
+ client.executeQuery("CREATE TABLE default.table2 (age int);");
+ }
+
+ private void verifyDataBaseAndTable() throws Exception {
+ client.existDatabase("default");
+ client.existTable("default.table1");
+ client.existTable("default.table2");
}
}
[10/13] tajo git commit: TAJO-1232: Implicit groupby queries with
LIMIT lead to wrong results. (jihoon)
Posted by ji...@apache.org.
TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/af4083bf
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/af4083bf
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/af4083bf
Branch: refs/heads/index_support
Commit: af4083bfd337bfd6fe8278b0363c987d19088e50
Parents: 5c852b7
Author: Jihoon Son <ji...@apache.org>
Authored: Thu Dec 18 16:50:09 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Thu Dec 18 16:50:09 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../org/apache/tajo/master/GlobalEngine.java | 8 ++++----
.../tajo/engine/query/TestGroupByQuery.java | 21 ++++++++++++++++++++
.../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 +
.../testGroupbyWithLimit1.result | 3 +++
.../testGroupbyWithLimit2.result | 3 +++
.../testGroupbyWithLimit3.result | 3 +++
.../org/apache/tajo/plan/LogicalPlanner.java | 7 +++++++
10 files changed, 47 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2230d0f..29b66d1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -112,6 +112,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results.
+ (jihoon)
+
TAJO-1254: Fix getProgress race conditions in Query. (jinho)
TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses.
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index a9624f8..71b1f9b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -204,10 +204,10 @@ public class GlobalEngine extends AbstractService {
}
private SubmitQueryResponse executeQueryInternal(QueryContext queryContext,
- Session session,
- LogicalPlan plan,
- String sql,
- String jsonExpr) throws Exception {
+ Session session,
+ LogicalPlan plan,
+ String sql,
+ String jsonExpr) throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index 3dd1219..edbe029 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -749,4 +749,25 @@ public class TestGroupByQuery extends QueryTestCaseBase {
ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal);
}
}
+
+ @Test
+ public final void testGroupbyWithLimit1() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testGroupbyWithLimit2() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testGroupbyWithLimit3() throws Exception {
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
new file mode 100644
index 0000000..06d96af
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit1.sql
@@ -0,0 +1 @@
+select sum(l_linenumber) from lineitem limit 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
new file mode 100644
index 0000000..1ddb171
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit2.sql
@@ -0,0 +1 @@
+select sum(l_linenumber) from lineitem limit 10;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
new file mode 100644
index 0000000..1bb7c7f
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
@@ -0,0 +1 @@
+select l_orderkey, sum(l_linenumber) from lineitem group by l_orderkey limit 1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
new file mode 100644
index 0000000..23bbaea
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit1.result
@@ -0,0 +1,3 @@
+?sum
+-------------------------------
+7
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
new file mode 100644
index 0000000..23bbaea
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit2.result
@@ -0,0 +1,3 @@
+?sum
+-------------------------------
+7
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
new file mode 100644
index 0000000..cb1e141
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
@@ -0,0 +1,3 @@
+l_orderkey,?sum
+-------------------------------
+3,3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/af4083bf/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index df45c3a..1a426e0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -716,6 +716,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
LogicalPlan plan = context.plan;
QueryBlock block = context.queryBlock;
+
+ // The limit operation must affect to the number of results, not the number of input records.
+ // Thus, the aggregation must be carried out before the limit operation.
+ if (child.getType() == NodeType.LIMIT) {
+ child = ((LimitNode)child).getChild();
+ }
+
GroupbyNode groupbyNode = context.plan.createNode(GroupbyNode.class);
groupbyNode.setChild(child);
groupbyNode.setInSchema(child.getOutSchema());
[11/13] tajo git commit: TAJO-1247: Store type 'TEXTFILE' should be
TEXT while keeping enum 'TEXTFILE' in protobuf. (DaeMyung Kang via hyunsik)
Posted by ji...@apache.org.
TAJO-1247: Store type 'TEXTFILE' should be TEXT while keeping enum 'TEXTFILE' in protobuf. (DaeMyung Kang via hyunsik)
Closes #302
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b7c8cf72
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b7c8cf72
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b7c8cf72
Branch: refs/heads/index_support
Commit: b7c8cf7239483c6b44925175658f21dbbf252665
Parents: af4083b
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Dec 19 10:07:53 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Dec 19 10:10:52 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../org/apache/tajo/catalog/CatalogUtil.java | 24 ++++++++++++++++----
.../org/apache/tajo/catalog/DDLBuilder.java | 6 ++---
.../apache/tajo/catalog/store/HCatalogUtil.java | 3 ++-
.../TestSelectQuery/datetime_table_ddl.sql | 2 +-
.../datetime_table_timezoned_ddl.sql | 2 +-
.../src/main/sphinx/table_management/csv.rst | 4 ++--
.../sphinx/table_management/table_overview.rst | 4 ++--
tajo-docs/src/main/sphinx/time_zone.rst | 6 ++---
9 files changed, 37 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 29b66d1..54a4ef4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,9 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1247: Store type 'TEXTFILE' should be TEXT while keeping enum 'TEXTFILE' in protobuf.
+ (DaeMyung Kang via hyunsik)
+
TAJO-1221: HA TajoClient should not connect TajoMaster at the first.
(jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index cbcac63..bb15ec1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -18,7 +18,6 @@
package org.apache.tajo.catalog;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.DataTypeUtil;
@@ -32,11 +31,9 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.storage.StorageConstants;
-import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
-import org.mortbay.util.ajax.JSON;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -52,6 +49,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type;
public class CatalogUtil {
+ public static final String TEXTFILE_NAME = "TEXT";
/**
* Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name.
*
@@ -264,6 +262,24 @@ public class CatalogUtil {
return sb.toString();
}
+ public static String getStoreTypeString(final StoreType type) {
+ if (type == StoreType.TEXTFILE) {
+ return TEXTFILE_NAME;
+ } else if (type == StoreType.CSV ||
+ type == StoreType.RAW ||
+ type == StoreType.ROWFILE ||
+ type == StoreType.RCFILE ||
+ type == StoreType.PARQUET ||
+ type == StoreType.SEQUENCEFILE ||
+ type == StoreType.AVRO ||
+ type == StoreType.JSON ||
+ type == StoreType.HBASE) {
+ return type.name();
+ } else {
+ return null;
+ }
+ }
+
public static StoreType getStoreType(final String typeStr) {
if (typeStr.equalsIgnoreCase(StoreType.CSV.name())) {
return StoreType.CSV;
@@ -279,7 +295,7 @@ public class CatalogUtil {
return StoreType.SEQUENCEFILE;
} else if (typeStr.equalsIgnoreCase(StoreType.AVRO.name())) {
return StoreType.AVRO;
- } else if (typeStr.equalsIgnoreCase(StoreType.TEXTFILE.name())) {
+ } else if (typeStr.equalsIgnoreCase(TEXTFILE_NAME)) {
return StoreType.TEXTFILE;
} else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) {
return StoreType.JSON;
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
index bd7364a..978092a 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java
@@ -31,7 +31,7 @@ public class DDLBuilder {
sb.append("--\n")
.append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
- .append(" Storage: ").append(desc.getMeta().getStoreType().name());
+ .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType()));
sb.append("\n-- Path: ").append(desc.getPath());
sb.append("\n--\n");
sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
@@ -54,7 +54,7 @@ public class DDLBuilder {
sb.append("--\n")
.append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;")
- .append(" Storage: ").append(desc.getMeta().getStoreType().name());
+ .append(" Storage: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType()));
sb.append("\n--\n");
sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName()));
buildSchema(sb, desc.getSchema());
@@ -91,7 +91,7 @@ public class DDLBuilder {
}
private static void buildUsingClause(StringBuilder sb, TableMeta meta) {
- sb.append(" USING " + meta.getStoreType().name());
+ sb.append(" USING " + CatalogUtil.getStoreTypeString(meta.getStoreType()));
}
private static void buildWithClause(StringBuilder sb, TableMeta meta) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
index c7f2590..ff2a2c4 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java
@@ -31,6 +31,7 @@ import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.thrift.TException;
import parquet.hadoop.mapred.DeprecatedParquetOutputFormat;
@@ -128,7 +129,7 @@ public class HCatalogUtil {
String outputFormatClass = fileFormatArrary[fileFormatArrary.length-1];
if(outputFormatClass.equals(HiveIgnoreKeyTextOutputFormat.class.getSimpleName())) {
- return CatalogProtos.StoreType.TEXTFILE.name();
+ return CatalogUtil.TEXTFILE_NAME;
} else if(outputFormatClass.equals(HiveSequenceFileOutputFormat.class.getSimpleName())) {
return CatalogProtos.StoreType.SEQUENCEFILE.name();
} else if(outputFormatClass.equals(RCFileOutputFormat.class.getSimpleName())) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql
index beb19b7..d2c97c5 100644
--- a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql
@@ -1,4 +1,4 @@
CREATE EXTERNAL TABLE ${0} (
t_timestamp TIMESTAMP,
t_date DATE
-) USING TEXTFILE LOCATION ${table.path}
\ No newline at end of file
+) USING TEXT LOCATION ${table.path}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql
index 11014d8..131e619 100644
--- a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql
@@ -1,4 +1,4 @@
CREATE EXTERNAL TABLE ${0} (
t_timestamp TIMESTAMP,
t_date DATE
-) USING TEXTFILE WITH ('timezone' = 'GMT+9') LOCATION ${table.path}
\ No newline at end of file
+) USING TEXT WITH ('timezone' = 'GMT+9') LOCATION ${table.path}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/table_management/csv.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/table_management/csv.rst b/tajo-docs/src/main/sphinx/table_management/csv.rst
index 37dc2ba..53c6e1d 100644
--- a/tajo-docs/src/main/sphinx/table_management/csv.rst
+++ b/tajo-docs/src/main/sphinx/table_management/csv.rst
@@ -110,6 +110,6 @@ clause in a Hive's ``CREATE TABLE`` statement as follows:
CREATE TABLE table1 (id int, name string, score float, type string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
- STORED AS TEXTFILE
+ STORED AS TEXT
-To the best of our knowledge, there is not way to specify a custom NULL character in Hive.
\ No newline at end of file
+To the best of our knowledge, there is not way to specify a custom NULL character in Hive.
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/table_management/table_overview.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst
index 704d6aa..3d933c2 100644
--- a/tajo-docs/src/main/sphinx/table_management/table_overview.rst
+++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst
@@ -57,7 +57,7 @@ You can specify a table time zone as follows:
CREATE EXTERNAL TABLE table1 (
t_timestamp TIMESTAMP,
t_date DATE
- ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
+ ) USING TEXT WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
-In order to learn time zone, please refer to :doc:`/time_zone`.
\ No newline at end of file
+In order to learn time zone, please refer to :doc:`/time_zone`.
http://git-wip-us.apache.org/repos/asf/tajo/blob/b7c8cf72/tajo-docs/src/main/sphinx/time_zone.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/time_zone.rst b/tajo-docs/src/main/sphinx/time_zone.rst
index 8225dee..101012f 100644
--- a/tajo-docs/src/main/sphinx/time_zone.rst
+++ b/tajo-docs/src/main/sphinx/time_zone.rst
@@ -36,7 +36,7 @@ You can specify a table time zone as follows:
CREATE EXTERNAL TABLE table1 (
t_timestamp TIMESTAMP,
t_date DATE
- ) USING TEXTFILE WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
+ ) USING TEXT WITH('timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
In order to learn table properties, please refer to :doc:`/table_management/table_overview`.
@@ -102,7 +102,7 @@ In order to register the table, we should put a table property ``'timezone'='Asi
CREATE EXTERNAL TABLE table1 (
t_timestamp TIMESTAMP,
t_date DATE
- ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
+ ) USING TEXT WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/'
By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone.
@@ -126,4 +126,4 @@ In addition, users can set client-side time zone by setting a session variable '
----------------------------------
1980-04-01 01:50:30.01, 1980-04-01
1980-04-01 01:50:30 , 1980-04-01
- 1980-04-01 01:50:30 , 1980-04-01
\ No newline at end of file
+ 1980-04-01 01:50:30 , 1980-04-01
[13/13] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/94faf6ec
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/94faf6ec
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/94faf6ec
Branch: refs/heads/index_support
Commit: 94faf6ec86ff596150c3c826813688f9f246c11a
Parents: 5893d2c 3413107
Author: Jihoon Son <ji...@apache.org>
Authored: Fri Dec 19 21:49:03 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri Dec 19 21:49:03 2014 +0900
----------------------------------------------------------------------
CHANGES | 19 +
.../tajo/catalog/AbstractCatalogClient.java | 2 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 24 +-
.../org/apache/tajo/catalog/DDLBuilder.java | 6 +-
.../apache/tajo/catalog/store/HCatalogUtil.java | 3 +-
.../org/apache/tajo/cli/tools/TajoAdmin.java | 2 +-
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 2 +-
.../apache/tajo/client/SessionConnection.java | 2 +-
.../org/apache/tajo/client/TajoClientImpl.java | 5 +-
.../apache/tajo/client/TajoHAClientUtil.java | 20 +-
.../java/org/apache/tajo/QueryIdFactory.java | 12 +-
.../org/apache/tajo/QueryUnitAttemptId.java | 94 --
.../main/java/org/apache/tajo/QueryUnitId.java | 90 --
.../java/org/apache/tajo/TaskAttemptId.java | 94 ++
.../src/main/java/org/apache/tajo/TaskId.java | 90 ++
.../java/org/apache/tajo/ha/HAConstants.java | 27 +
.../java/org/apache/tajo/ha/HAServiceUtil.java | 292 ++++++
.../org/apache/tajo/util/HAServiceUtil.java | 293 ------
.../java/org/apache/tajo/util/TajoIdUtils.java | 8 +-
.../apache/tajo/validation/PathValidator.java | 2 +-
tajo-common/src/main/proto/TajoIdProtos.proto | 6 +-
.../apache/tajo/validation/TestValidators.java | 12 +
.../physical/HashShuffleFileWriteExec.java | 3 +-
.../engine/planner/physical/PhysicalExec.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 2 +-
.../tajo/engine/query/QueryUnitRequest.java | 51 --
.../tajo/engine/query/QueryUnitRequestImpl.java | 328 -------
.../apache/tajo/engine/query/TaskRequest.java | 51 ++
.../tajo/engine/query/TaskRequestImpl.java | 328 +++++++
.../tajo/master/DefaultTaskScheduler.java | 162 ++--
.../org/apache/tajo/master/GlobalEngine.java | 8 +-
.../apache/tajo/master/LazyTaskScheduler.java | 46 +-
.../master/NonForwardQueryResultScanner.java | 6 +-
.../apache/tajo/master/TajoContainerProxy.java | 6 +-
.../event/ContainerAllocatorEventType.java | 2 +-
.../tajo/master/event/LocalTaskEvent.java | 8 +-
.../event/QueryUnitAttemptScheduleEvent.java | 87 --
.../tajo/master/event/SubQueryEventType.java | 2 +-
.../tajo/master/event/SubQueryTaskEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 4 +-
.../tajo/master/event/TaskAttemptEvent.java | 8 +-
.../master/event/TaskAttemptScheduleEvent.java | 4 +-
.../event/TaskAttemptStatusUpdateEvent.java | 4 +-
.../event/TaskAttemptToSchedulerEvent.java | 87 ++
.../tajo/master/event/TaskCompletionEvent.java | 4 +-
.../org/apache/tajo/master/event/TaskEvent.java | 8 +-
.../tajo/master/event/TaskFatalErrorEvent.java | 6 +-
.../tajo/master/event/TaskRequestEvent.java | 9 +-
.../tajo/master/event/TaskTAttemptEvent.java | 10 +-
.../tajo/master/ha/HAServiceHDFSImpl.java | 100 +-
.../apache/tajo/master/querymaster/Query.java | 2 +-
.../master/querymaster/QueryJobManager.java | 5 +-
.../tajo/master/querymaster/QueryMaster.java | 2 +-
.../querymaster/QueryMasterManagerService.java | 22 +-
.../master/querymaster/QueryMasterTask.java | 16 +-
.../tajo/master/querymaster/QueryUnit.java | 907 -------------------
.../master/querymaster/QueryUnitAttempt.java | 443 ---------
.../tajo/master/querymaster/Repartitioner.java | 14 +-
.../tajo/master/querymaster/SubQuery.java | 72 +-
.../apache/tajo/master/querymaster/Task.java | 907 +++++++++++++++++++
.../tajo/master/querymaster/TaskAttempt.java | 443 +++++++++
.../main/java/org/apache/tajo/util/JSPUtil.java | 84 +-
.../apache/tajo/util/history/HistoryReader.java | 22 +-
.../apache/tajo/util/history/HistoryWriter.java | 4 +-
.../tajo/util/history/QueryUnitHistory.java | 167 ----
.../tajo/util/history/SubQueryHistory.java | 22 +-
.../apache/tajo/util/history/TaskHistory.java | 167 ++++
.../tajo/worker/ExecutionBlockContext.java | 12 +-
.../java/org/apache/tajo/worker/FetchImpl.java | 19 +-
.../apache/tajo/worker/InterDataRetriever.java | 8 +-
.../tajo/worker/TajoResourceAllocator.java | 2 +-
.../java/org/apache/tajo/worker/TajoWorker.java | 1 +
.../tajo/worker/TajoWorkerManagerService.java | 8 +-
.../main/java/org/apache/tajo/worker/Task.java | 31 +-
.../apache/tajo/worker/TaskAttemptContext.java | 12 +-
.../org/apache/tajo/worker/TaskHistory.java | 18 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 18 +-
.../apache/tajo/worker/TaskRunnerHistory.java | 16 +-
.../apache/tajo/worker/TaskRunnerManager.java | 10 +-
.../tajo/worker/WorkerHeartbeatService.java | 2 +-
.../retriever/AdvancedDataRetriever.java | 14 +-
.../ConnectivityCheckerRuleForTajoWorker.java | 2 +-
.../src/main/proto/QueryMasterProtocol.proto | 2 +-
.../src/main/proto/TajoWorkerProtocol.proto | 22 +-
.../main/resources/webapps/admin/querytasks.jsp | 70 +-
.../main/resources/webapps/admin/queryunit.jsp | 134 ---
.../src/main/resources/webapps/admin/task.jsp | 134 +++
.../resources/webapps/worker/querytasks.jsp | 70 +-
.../main/resources/webapps/worker/queryunit.jsp | 175 ----
.../src/main/resources/webapps/worker/task.jsp | 174 ++++
.../resources/webapps/worker/taskdetail.jsp | 14 +-
.../resources/webapps/worker/taskhistory.jsp | 6 +-
.../src/main/resources/webapps/worker/tasks.jsp | 14 +-
.../apache/tajo/LocalTajoTestingUtility.java | 10 +-
.../org/apache/tajo/TajoTestingCluster.java | 3 +
.../org/apache/tajo/TestQueryIdFactory.java | 6 +-
.../test/java/org/apache/tajo/TestTajoIds.java | 32 +-
.../org/apache/tajo/client/TestTajoClient.java | 12 +-
.../planner/physical/TestBNLJoinExec.java | 4 +-
.../planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 2 +-
.../physical/TestFullOuterHashJoinExec.java | 8 +-
.../physical/TestFullOuterMergeJoinExec.java | 12 +-
.../planner/physical/TestHashAntiJoinExec.java | 2 +-
.../planner/physical/TestHashJoinExec.java | 4 +-
.../planner/physical/TestHashSemiJoinExec.java | 2 +-
.../physical/TestLeftOuterHashJoinExec.java | 10 +-
.../physical/TestLeftOuterNLJoinExec.java | 10 +-
.../planner/physical/TestMergeJoinExec.java | 2 +-
.../engine/planner/physical/TestNLJoinExec.java | 4 +-
.../planner/physical/TestPhysicalPlanner.java | 60 +-
.../physical/TestProgressExternalSortExec.java | 2 +-
.../physical/TestRightOuterHashJoinExec.java | 6 +-
.../physical/TestRightOuterMergeJoinExec.java | 12 +-
.../engine/planner/physical/TestSortExec.java | 2 +-
.../tajo/engine/query/TestGroupByQuery.java | 25 +-
.../apache/tajo/master/TestRepartitioner.java | 21 +-
.../tajo/master/ha/TestHAServiceHDFSImpl.java | 115 ++-
.../querymaster/TestIntermediateEntry.java | 2 +-
.../querymaster/TestQueryUnitStatusUpdate.java | 194 ----
.../querymaster/TestTaskStatusUpdate.java | 194 ++++
.../java/org/apache/tajo/util/TestJSPUtil.java | 54 +-
.../util/history/TestHistoryWriterReader.java | 39 +-
.../org/apache/tajo/worker/TestHistory.java | 8 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 4 +-
.../worker/dataserver/TestHttpDataServer.java | 12 +-
.../TestGroupByQuery/testGroupbyWithLimit1.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit2.sql | 1 +
.../TestGroupByQuery/testGroupbyWithLimit3.sql | 1 +
.../queries/TestQueryUnitStatusUpdate/case1.sql | 1 -
.../queries/TestQueryUnitStatusUpdate/case2.sql | 5 -
.../queries/TestQueryUnitStatusUpdate/case3.sql | 10 -
.../TestSelectQuery/datetime_table_ddl.sql | 2 +-
.../datetime_table_timezoned_ddl.sql | 2 +-
.../queries/TestTaskStatusUpdate/case1.sql | 1 +
.../queries/TestTaskStatusUpdate/case2.sql | 5 +
.../queries/TestTaskStatusUpdate/case3.sql | 10 +
.../testGroupbyWithLimit1.result | 3 +
.../testGroupbyWithLimit2.result | 3 +
.../testGroupbyWithLimit3.result | 3 +
tajo-dist/pom.xml | 4 +-
.../configuration/catalog_configuration.rst | 4 +-
.../src/main/sphinx/table_management/csv.rst | 4 +-
.../sphinx/table_management/table_overview.rst | 4 +-
tajo-docs/src/main/sphinx/time_zone.rst | 6 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 7 +
.../tajo/pullserver/HttpDataServerHandler.java | 2 +-
.../retriever/AdvancedDataRetriever.java | 16 +-
.../org/apache/tajo/storage/StorageManager.java | 6 +-
.../org/apache/tajo/storage/StorageUtil.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 6 +-
.../tajo/storage/hbase/HBasePutAppender.java | 4 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 2 +-
.../tajo/storage/hbase/HFileAppender.java | 8 +-
.../java/org/apache/tajo/storage/CSVFile.java | 4 +-
.../org/apache/tajo/storage/FileAppender.java | 6 +-
.../apache/tajo/storage/FileStorageManager.java | 8 +-
.../tajo/storage/HashShuffleAppender.java | 12 +-
.../storage/HashShuffleAppenderManager.java | 6 +-
.../java/org/apache/tajo/storage/RawFile.java | 4 +-
.../java/org/apache/tajo/storage/RowFile.java | 4 +-
.../apache/tajo/storage/avro/AvroAppender.java | 4 +-
.../tajo/storage/parquet/ParquetAppender.java | 4 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 4 +-
.../sequencefile/SequenceFileAppender.java | 4 +-
.../tajo/storage/text/DelimitedTextFile.java | 5 +-
166 files changed, 3991 insertions(+), 3810 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index ac00c15,6db76ae..a4e49f7
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@@ -987,15 -987,9 +987,15 @@@ public class TestPhysicalPlanner
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employee.getMeta(),
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
+ Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee");
+ if (sm.getFileSystem().exists(indexPath)) {
+ sm.getFileSystem().delete(indexPath, true);
+ }
+
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
+ ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(createIndexStmt[0]);
LogicalPlan plan = planner.createPlan(defaultContext, context);
LogicalNode rootNode = optimizer.optimize(plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/94faf6ec/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
[08/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
deleted file mode 100644
index 6e0d9fd..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.event;
-
-import com.google.protobuf.RpcCallback;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProto;
-import org.apache.tajo.master.querymaster.QueryUnitAttempt;
-import org.apache.tajo.master.container.TajoContainerId;
-
-public class QueryUnitAttemptScheduleEvent extends TaskSchedulerEvent {
- private final QueryUnitAttemptScheduleContext context;
- private final QueryUnitAttempt queryUnitAttempt;
-
- public QueryUnitAttemptScheduleEvent(EventType eventType, ExecutionBlockId executionBlockId,
- QueryUnitAttemptScheduleContext context, QueryUnitAttempt queryUnitAttempt) {
- super(eventType, executionBlockId);
- this.context = context;
- this.queryUnitAttempt = queryUnitAttempt;
- }
-
- public QueryUnitAttempt getQueryUnitAttempt() {
- return queryUnitAttempt;
- }
-
- public QueryUnitAttemptScheduleContext getContext() {
- return context;
- }
-
- public static class QueryUnitAttemptScheduleContext {
- private TajoContainerId containerId;
- private String host;
- private RpcCallback<QueryUnitRequestProto> callback;
-
- public QueryUnitAttemptScheduleContext() {
-
- }
-
- public QueryUnitAttemptScheduleContext(TajoContainerId containerId,
- String host,
- RpcCallback<QueryUnitRequestProto> callback) {
- this.containerId = containerId;
- this.host = host;
- this.callback = callback;
- }
-
- public TajoContainerId getContainerId() {
- return containerId;
- }
-
- public void setContainerId(TajoContainerId containerId) {
- this.containerId = containerId;
- }
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public RpcCallback<QueryUnitRequestProto> getCallback() {
- return callback;
- }
-
- public void setCallback(RpcCallback<QueryUnitRequestProto> callback) {
- this.callback = callback;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryEventType.java
index 8003ef3..79b6e2e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryEventType.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryEventType.java
@@ -30,7 +30,7 @@ public enum SubQueryEventType {
SQ_KILL,
SQ_LAUNCH,
- // Producer: QueryUnit
+ // Producer: Task
SQ_TASK_COMPLETED,
SQ_FAILED,
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryTaskEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryTaskEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryTaskEvent.java
index 0502534..816bc48 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryTaskEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryTaskEvent.java
@@ -18,22 +18,22 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.master.TaskState;
/**
* Event Class: From Task to SubQuery
*/
public class SubQueryTaskEvent extends SubQueryEvent {
- private QueryUnitId taskId;
+ private TaskId taskId;
private TaskState state;
- public SubQueryTaskEvent(QueryUnitId taskId, TaskState state) {
+ public SubQueryTaskEvent(TaskId taskId, TaskState state) {
super(taskId.getExecutionBlockId(), SubQueryEventType.SQ_TASK_COMPLETED);
this.taskId = taskId;
this.state = state;
}
- public QueryUnitId getTaskId() {
+ public TaskId getTaskId() {
return this.taskId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
index 3b9edcb..1611370 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
@@ -18,7 +18,7 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.container.TajoContainerId;
@@ -26,7 +26,7 @@ public class TaskAttemptAssignedEvent extends TaskAttemptEvent {
private final TajoContainerId cId;
private final WorkerConnectionInfo workerConnectionInfo;
- public TaskAttemptAssignedEvent(QueryUnitAttemptId id, TajoContainerId cId,
+ public TaskAttemptAssignedEvent(TaskAttemptId id, TajoContainerId cId,
WorkerConnectionInfo connectionInfo) {
super(id, TaskAttemptEventType.TA_ASSIGNED);
this.cId = cId;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEvent.java
index f2df144..1b84de0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptEvent.java
@@ -19,18 +19,18 @@
package org.apache.tajo.master.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
public class TaskAttemptEvent extends AbstractEvent<TaskAttemptEventType> {
- private final QueryUnitAttemptId id;
+ private final TaskAttemptId id;
- public TaskAttemptEvent(QueryUnitAttemptId id,
+ public TaskAttemptEvent(TaskAttemptId id,
TaskAttemptEventType taskAttemptEventType) {
super(taskAttemptEventType);
this.id = id;
}
- public QueryUnitAttemptId getTaskAttemptId() {
+ public TaskAttemptId getTaskAttemptId() {
return this.id;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptScheduleEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptScheduleEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptScheduleEvent.java
index 8f153af..3274ef7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptScheduleEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptScheduleEvent.java
@@ -19,13 +19,13 @@
package org.apache.tajo.master.event;
import org.apache.hadoop.conf.Configuration;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
public class TaskAttemptScheduleEvent extends TaskAttemptEvent {
private Configuration conf;
public TaskAttemptScheduleEvent(final Configuration conf,
- final QueryUnitAttemptId id,
+ final TaskAttemptId id,
final TaskAttemptEventType taskAttemptEventType) {
super(id, taskAttemptEventType);
this.conf = conf;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
index d980e05..8c5f016 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptStatusUpdateEvent.java
@@ -18,13 +18,13 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskStatusProto;
public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent {
private final TaskStatusProto status;
- public TaskAttemptStatusUpdateEvent(final QueryUnitAttemptId id,
+ public TaskAttemptStatusUpdateEvent(final TaskAttemptId id,
TaskStatusProto status) {
super(id, TaskAttemptEventType.TA_UPDATE);
this.status = status;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
new file mode 100644
index 0000000..91ef942
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptToSchedulerEvent.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.event;
+
+import com.google.protobuf.RpcCallback;
+import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.master.querymaster.TaskAttempt;
+import org.apache.tajo.master.container.TajoContainerId;
+
+public class TaskAttemptToSchedulerEvent extends TaskSchedulerEvent {
+ private final TaskAttemptScheduleContext context;
+ private final TaskAttempt taskAttempt;
+
+ public TaskAttemptToSchedulerEvent(EventType eventType, ExecutionBlockId executionBlockId,
+ TaskAttemptScheduleContext context, TaskAttempt taskAttempt) {
+ super(eventType, executionBlockId);
+ this.context = context;
+ this.taskAttempt = taskAttempt;
+ }
+
+ public TaskAttempt getTaskAttempt() {
+ return taskAttempt;
+ }
+
+ public TaskAttemptScheduleContext getContext() {
+ return context;
+ }
+
+ public static class TaskAttemptScheduleContext {
+ private TajoContainerId containerId;
+ private String host;
+ private RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback;
+
+ public TaskAttemptScheduleContext() {
+
+ }
+
+ public TaskAttemptScheduleContext(TajoContainerId containerId,
+ String host,
+ RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback) {
+ this.containerId = containerId;
+ this.host = host;
+ this.callback = callback;
+ }
+
+ public TajoContainerId getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(TajoContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public RpcCallback<TajoWorkerProtocol.TaskRequestProto> getCallback() {
+ return callback;
+ }
+
+ public void setCallback(RpcCallback<TajoWorkerProtocol.TaskRequestProto> callback) {
+ this.callback = callback;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
index 3ee389a..20204aa 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskCompletionEvent.java
@@ -18,14 +18,14 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
public class TaskCompletionEvent extends TaskAttemptEvent {
private TaskCompletionReport report;
public TaskCompletionEvent(TaskCompletionReport report) {
- super(new QueryUnitAttemptId(report.getId()), TaskAttemptEventType.TA_DONE);
+ super(new TaskAttemptId(report.getId()), TaskAttemptEventType.TA_DONE);
this.report = report;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskEvent.java
index 234491b..377a8e0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskEvent.java
@@ -19,17 +19,17 @@
package org.apache.tajo.master.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskId;
public class TaskEvent extends AbstractEvent<TaskEventType> {
- private final QueryUnitId id;
+ private final TaskId id;
- public TaskEvent(QueryUnitId id, TaskEventType taskEventType) {
+ public TaskEvent(TaskId id, TaskEventType taskEventType) {
super(taskEventType);
this.id = id;
}
- public QueryUnitId getTaskId() {
+ public TaskId getTaskId() {
return id;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
index a4d9900..03888bd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskFatalErrorEvent.java
@@ -18,19 +18,19 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.ipc.TajoWorkerProtocol.TaskFatalErrorReport;
public class TaskFatalErrorEvent extends TaskAttemptEvent {
private final String message;
public TaskFatalErrorEvent(TaskFatalErrorReport report) {
- super(new QueryUnitAttemptId(report.getId()),
+ super(new TaskAttemptId(report.getId()),
TaskAttemptEventType.TA_FATAL_ERROR);
this.message = report.getErrorMessage();
}
- public TaskFatalErrorEvent(QueryUnitAttemptId attemptId, String message) {
+ public TaskFatalErrorEvent(TaskAttemptId attemptId, String message) {
super(attemptId, TaskAttemptEventType.TA_FATAL_ERROR);
this.message = message;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
index 9e8e3dd..3f72ed9 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
@@ -21,7 +21,8 @@ package org.apache.tajo.master.event;
import com.google.protobuf.RpcCallback;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProto;
+import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto;
import org.apache.tajo.master.event.TaskRequestEvent.TaskRequestEventType;
import org.apache.tajo.master.container.TajoContainerId;
@@ -35,12 +36,12 @@ public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
private final TajoContainerId containerId;
private final ExecutionBlockId executionBlockId;
- private final RpcCallback<QueryUnitRequestProto> callback;
+ private final RpcCallback<TaskRequestProto> callback;
public TaskRequestEvent(int workerId,
TajoContainerId containerId,
ExecutionBlockId executionBlockId,
- RpcCallback<QueryUnitRequestProto> callback) {
+ RpcCallback<TaskRequestProto> callback) {
super(TaskRequestEventType.TASK_REQ);
this.workerId = workerId;
this.containerId = containerId;
@@ -60,7 +61,7 @@ public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
return executionBlockId;
}
- public RpcCallback<QueryUnitRequestProto> getCallback() {
+ public RpcCallback<TajoWorkerProtocol.TaskRequestProto> getCallback() {
return this.callback;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/TaskTAttemptEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskTAttemptEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskTAttemptEvent.java
index 28654f0..a4f120c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskTAttemptEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskTAttemptEvent.java
@@ -18,17 +18,17 @@
package org.apache.tajo.master.event;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
public class TaskTAttemptEvent extends TaskEvent {
- private final QueryUnitAttemptId attemptId;
- public TaskTAttemptEvent(QueryUnitAttemptId attemptId,
+ private final TaskAttemptId attemptId;
+ public TaskTAttemptEvent(TaskAttemptId attemptId,
TaskEventType eventType) {
- super(attemptId.getQueryUnitId(), eventType);
+ super(attemptId.getTaskId(), eventType);
this.attemptId = attemptId;
}
- public QueryUnitAttemptId getTaskAttemptId() {
+ public TaskAttemptId getTaskAttemptId() {
return attemptId;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
index f4bd8a3..e7e2bc0 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
@@ -122,7 +122,7 @@ public class QueryMasterManagerService extends CompositeService
@Override
public void getTask(RpcController controller, TajoWorkerProtocol.GetTaskRequestProto request,
- RpcCallback<TajoWorkerProtocol.QueryUnitRequestProto> done) {
+ RpcCallback<TajoWorkerProtocol.TaskRequestProto> done) {
try {
ExecutionBlockId ebId = new ExecutionBlockId(request.getExecutionBlockId());
QueryMasterTask queryMasterTask = workerContext.getQueryMaster().getQueryMasterTask(ebId.getQueryId());
@@ -144,15 +144,15 @@ public class QueryMasterManagerService extends CompositeService
public void statusUpdate(RpcController controller, TajoWorkerProtocol.TaskStatusProto request,
RpcCallback<PrimitiveProtos.BoolProto> done) {
try {
- QueryId queryId = new QueryId(request.getId().getQueryUnitId().getExecutionBlockId().getQueryId());
- QueryUnitAttemptId attemptId = new QueryUnitAttemptId(request.getId());
+ QueryId queryId = new QueryId(request.getId().getTaskId().getExecutionBlockId().getQueryId());
+ TaskAttemptId attemptId = new TaskAttemptId(request.getId());
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(queryId);
if (queryMasterTask == null) {
queryMasterTask = queryMaster.getQueryMasterTask(queryId, true);
}
- SubQuery sq = queryMasterTask.getQuery().getSubQuery(attemptId.getQueryUnitId().getExecutionBlockId());
- QueryUnit task = sq.getQueryUnit(attemptId.getQueryUnitId());
- QueryUnitAttempt attempt = task.getAttempt(attemptId.getId());
+ SubQuery sq = queryMasterTask.getQuery().getSubQuery(attemptId.getTaskId().getExecutionBlockId());
+ Task task = sq.getTask(attemptId.getTaskId());
+ TaskAttempt attempt = task.getAttempt(attemptId.getId());
if(LOG.isDebugEnabled()){
LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name()));
@@ -161,10 +161,10 @@ public class QueryMasterManagerService extends CompositeService
if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) {
LOG.warn(attemptId + " Killed");
attempt.handle(
- new TaskAttemptEvent(new QueryUnitAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED));
+ new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED));
} else {
queryMasterTask.getEventHandler().handle(
- new TaskAttemptStatusUpdateEvent(new QueryUnitAttemptId(request.getId()), request));
+ new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request));
}
done.run(TajoWorker.TRUE_PROTO);
} catch (Exception e) {
@@ -185,11 +185,11 @@ public class QueryMasterManagerService extends CompositeService
RpcCallback<PrimitiveProtos.BoolProto> done) {
try {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
- new QueryId(report.getId().getQueryUnitId().getExecutionBlockId().getQueryId()));
+ new QueryId(report.getId().getTaskId().getExecutionBlockId().getQueryId()));
if (queryMasterTask != null) {
queryMasterTask.handleTaskFailed(report);
} else {
- LOG.warn("No QueryMasterTask: " + new QueryUnitAttemptId(report.getId()));
+ LOG.warn("No QueryMasterTask: " + new TaskAttemptId(report.getId()));
}
done.run(TajoWorker.TRUE_PROTO);
} catch (Exception e) {
@@ -203,7 +203,7 @@ public class QueryMasterManagerService extends CompositeService
RpcCallback<PrimitiveProtos.BoolProto> done) {
try {
QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(
- new QueryId(report.getId().getQueryUnitId().getExecutionBlockId().getQueryId()));
+ new QueryId(report.getId().getTaskId().getExecutionBlockId().getQueryId()));
if (queryMasterTask != null) {
queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 8f63416..9ab4f0a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -285,12 +285,12 @@ public class QueryMasterTask extends CompositeService {
private class TaskEventDispatcher
implements EventHandler<TaskEvent> {
public void handle(TaskEvent event) {
- QueryUnitId taskId = event.getTaskId();
+ TaskId taskId = event.getTaskId();
if(LOG.isDebugEnabled()) {
LOG.debug("TaskEventDispatcher>" + taskId + "," + event.getType());
}
- QueryUnit task = query.getSubQuery(taskId.getExecutionBlockId()).
- getQueryUnit(taskId);
+ Task task = query.getSubQuery(taskId.getExecutionBlockId()).
+ getTask(taskId);
task.handle(event);
}
}
@@ -298,10 +298,10 @@ public class QueryMasterTask extends CompositeService {
private class TaskAttemptEventDispatcher
implements EventHandler<TaskAttemptEvent> {
public void handle(TaskAttemptEvent event) {
- QueryUnitAttemptId attemptId = event.getTaskAttemptId();
- SubQuery subQuery = query.getSubQuery(attemptId.getQueryUnitId().getExecutionBlockId());
- QueryUnit task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
- QueryUnitAttempt attempt = task.getAttempt(attemptId);
+ TaskAttemptId attemptId = event.getTaskAttemptId();
+ SubQuery subQuery = query.getSubQuery(attemptId.getTaskId().getExecutionBlockId());
+ Task task = subQuery.getTask(attemptId.getTaskId());
+ TaskAttempt attempt = task.getAttempt(attemptId);
attempt.handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
deleted file mode 100644
index 75402c2..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
+++ /dev/null
@@ -1,907 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.querymaster;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.state.*;
-import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.QueryUnitId;
-import org.apache.tajo.TajoProtos.TaskAttemptState;
-import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.ipc.TajoWorkerProtocol.FailureIntermediateProto;
-import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto;
-import org.apache.tajo.master.FragmentPair;
-import org.apache.tajo.master.TaskState;
-import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
-import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.storage.DataLocation;
-import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.storage.fragment.Fragment;
-import org.apache.tajo.storage.fragment.FragmentConvertor;
-import org.apache.tajo.util.Pair;
-import org.apache.tajo.util.TajoIdUtils;
-import org.apache.tajo.util.history.QueryUnitHistory;
-import org.apache.tajo.worker.FetchImpl;
-
-import java.net.URI;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
-
-public class QueryUnit implements EventHandler<TaskEvent> {
- /** Class Logger */
- private static final Log LOG = LogFactory.getLog(QueryUnit.class);
-
- private final Configuration systemConf;
- private QueryUnitId taskId;
- private EventHandler eventHandler;
- private StoreTableNode store = null;
- private LogicalNode plan = null;
- private List<ScanNode> scan;
-
- private Map<String, Set<FragmentProto>> fragMap;
- private Map<String, Set<FetchImpl>> fetchMap;
-
- private int totalFragmentNum;
-
- private List<ShuffleFileOutput> shuffleFileOutputs;
- private TableStats stats;
- private final boolean isLeafTask;
- private List<IntermediateEntry> intermediateData;
-
- private Map<QueryUnitAttemptId, QueryUnitAttempt> attempts;
- private final int maxAttempts = 3;
- private Integer nextAttempt = -1;
- private QueryUnitAttemptId lastAttemptId;
-
- private QueryUnitAttemptId successfulAttempt;
- private String succeededHost;
- private int succeededHostPort;
- private int succeededPullServerPort;
-
- private int failedAttempts;
- private int finishedAttempts; // finish are total of success, failed and killed
-
- private long launchTime;
- private long finishTime;
-
- private List<DataLocation> dataLocations = Lists.newArrayList();
-
- private static final AttemptKilledTransition ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition();
-
- private QueryUnitHistory finalQueryUnitHistory;
-
- protected static final StateMachineFactory
- <QueryUnit, TaskState, TaskEventType, TaskEvent> stateMachineFactory =
- new StateMachineFactory <QueryUnit, TaskState, TaskEventType, TaskEvent>(TaskState.NEW)
-
- // Transitions from NEW state
- .addTransition(TaskState.NEW, TaskState.SCHEDULED,
- TaskEventType.T_SCHEDULE,
- new InitialScheduleTransition())
- .addTransition(TaskState.NEW, TaskState.KILLED,
- TaskEventType.T_KILL,
- new KillNewTaskTransition())
-
- // Transitions from SCHEDULED state
- .addTransition(TaskState.SCHEDULED, TaskState.RUNNING,
- TaskEventType.T_ATTEMPT_LAUNCHED,
- new AttemptLaunchedTransition())
- .addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT,
- TaskEventType.T_KILL,
- new KillTaskTransition())
-
- // Transitions from RUNNING state
- .addTransition(TaskState.RUNNING, TaskState.RUNNING,
- TaskEventType.T_ATTEMPT_LAUNCHED)
- .addTransition(TaskState.RUNNING, TaskState.SUCCEEDED,
- TaskEventType.T_ATTEMPT_SUCCEEDED,
- new AttemptSucceededTransition())
- .addTransition(TaskState.RUNNING, TaskState.KILL_WAIT,
- TaskEventType.T_KILL,
- new KillTaskTransition())
- .addTransition(TaskState.RUNNING,
- EnumSet.of(TaskState.RUNNING, TaskState.FAILED),
- TaskEventType.T_ATTEMPT_FAILED,
- new AttemptFailedOrRetryTransition())
-
- // Transitions from KILL_WAIT state
- .addTransition(TaskState.KILL_WAIT, TaskState.KILLED,
- TaskEventType.T_ATTEMPT_KILLED,
- ATTEMPT_KILLED_TRANSITION)
- .addTransition(TaskState.KILL_WAIT, TaskState.KILL_WAIT,
- TaskEventType.T_ATTEMPT_LAUNCHED,
- new KillTaskTransition())
- .addTransition(TaskState.KILL_WAIT, TaskState.FAILED,
- TaskEventType.T_ATTEMPT_FAILED,
- new AttemptFailedTransition())
- .addTransition(TaskState.KILL_WAIT, TaskState.KILLED,
- TaskEventType.T_ATTEMPT_SUCCEEDED,
- ATTEMPT_KILLED_TRANSITION)
- // Ignore-able transitions.
- .addTransition(TaskState.KILL_WAIT, TaskState.KILL_WAIT,
- EnumSet.of(
- TaskEventType.T_KILL,
- TaskEventType.T_SCHEDULE))
-
- // Transitions from SUCCEEDED state
- // Ignore-able transitions
- .addTransition(TaskState.SUCCEEDED, TaskState.SUCCEEDED,
- EnumSet.of(TaskEventType.T_KILL,
- TaskEventType.T_ATTEMPT_KILLED, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
-
- // Transitions from FAILED state
- // Ignore-able transitions
- .addTransition(TaskState.FAILED, TaskState.FAILED,
- EnumSet.of(TaskEventType.T_KILL,
- TaskEventType.T_ATTEMPT_KILLED, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
-
- // Transitions from KILLED state
- .addTransition(TaskState.KILLED, TaskState.KILLED, TaskEventType.T_ATTEMPT_KILLED, new KillTaskTransition())
- // Ignore-able transitions
- .addTransition(TaskState.KILLED, TaskState.KILLED,
- EnumSet.of(
- TaskEventType.T_KILL, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
-
- .installTopology();
-
- private final StateMachine<TaskState, TaskEventType, TaskEvent> stateMachine;
-
-
- private final Lock readLock;
- private final Lock writeLock;
- private QueryUnitAttemptScheduleContext scheduleContext;
-
- public QueryUnit(Configuration conf, QueryUnitAttemptScheduleContext scheduleContext,
- QueryUnitId id, boolean isLeafTask, EventHandler eventHandler) {
- this.systemConf = conf;
- this.taskId = id;
- this.eventHandler = eventHandler;
- this.isLeafTask = isLeafTask;
- scan = new ArrayList<ScanNode>();
- fetchMap = Maps.newHashMap();
- fragMap = Maps.newHashMap();
- shuffleFileOutputs = new ArrayList<ShuffleFileOutput>();
- attempts = Collections.emptyMap();
- lastAttemptId = null;
- nextAttempt = -1;
- failedAttempts = 0;
-
- ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- this.readLock = readWriteLock.readLock();
- this.writeLock = readWriteLock.writeLock();
- this.scheduleContext = scheduleContext;
-
- stateMachine = stateMachineFactory.make(this);
- totalFragmentNum = 0;
- }
-
- public boolean isLeafTask() {
- return this.isLeafTask;
- }
-
- public TaskState getState() {
- readLock.lock();
- try {
- return stateMachine.getCurrentState();
- } finally {
- readLock.unlock();
- }
- }
-
- public TaskAttemptState getLastAttemptStatus() {
- QueryUnitAttempt lastAttempt = getLastAttempt();
- if (lastAttempt != null) {
- return lastAttempt.getState();
- } else {
- return TaskAttemptState.TA_ASSIGNED;
- }
- }
-
- public QueryUnitHistory getQueryUnitHistory() {
- if (finalQueryUnitHistory != null) {
- if (finalQueryUnitHistory.getFinishTime() == 0) {
- finalQueryUnitHistory = makeQueryUnitHistory();
- }
- return finalQueryUnitHistory;
- } else {
- return makeQueryUnitHistory();
- }
- }
-
- private QueryUnitHistory makeQueryUnitHistory() {
- QueryUnitHistory queryUnitHistory = new QueryUnitHistory();
-
- QueryUnitAttempt lastAttempt = getLastAttempt();
- if (lastAttempt != null) {
- queryUnitHistory.setId(lastAttempt.getId().toString());
- queryUnitHistory.setState(lastAttempt.getState().toString());
- queryUnitHistory.setProgress(lastAttempt.getProgress());
- }
- queryUnitHistory.setHostAndPort(succeededHost + ":" + succeededHostPort);
- queryUnitHistory.setRetryCount(this.getRetryCount());
- queryUnitHistory.setLaunchTime(launchTime);
- queryUnitHistory.setFinishTime(finishTime);
-
- queryUnitHistory.setNumShuffles(getShuffleOutpuNum());
- if (!getShuffleFileOutputs().isEmpty()) {
- ShuffleFileOutput shuffleFileOutputs = getShuffleFileOutputs().get(0);
- if (queryUnitHistory.getNumShuffles() > 0) {
- queryUnitHistory.setShuffleKey("" + shuffleFileOutputs.getPartId());
- queryUnitHistory.setShuffleFileName(shuffleFileOutputs.getFileName());
- }
- }
-
- List<String> fragmentList = new ArrayList<String>();
- for (FragmentProto eachFragment : getAllFragments()) {
- try {
- Fragment fragment = FragmentConvertor.convert(systemConf, eachFragment);
- fragmentList.add(fragment.toString());
- } catch (Exception e) {
- LOG.error(e.getMessage());
- fragmentList.add("ERROR: " + eachFragment.getStoreType() + "," + eachFragment.getId() + ": " + e.getMessage());
- }
- }
- queryUnitHistory.setFragments(fragmentList.toArray(new String[]{}));
-
- List<String[]> fetchList = new ArrayList<String[]>();
- for (Map.Entry<String, Set<FetchImpl>> e : getFetchMap().entrySet()) {
- for (FetchImpl f : e.getValue()) {
- for (URI uri : f.getSimpleURIs()){
- fetchList.add(new String[] {e.getKey(), uri.toString()});
- }
- }
- }
-
- queryUnitHistory.setFetchs(fetchList.toArray(new String[][]{}));
-
- List<String> dataLocationList = new ArrayList<String>();
- for(DataLocation eachLocation: getDataLocations()) {
- dataLocationList.add(eachLocation.toString());
- }
-
- queryUnitHistory.setDataLocations(dataLocationList.toArray(new String[]{}));
- return queryUnitHistory;
- }
-
- public void setLogicalPlan(LogicalNode plan) {
- this.plan = plan;
-
- LogicalNode node = plan;
- ArrayList<LogicalNode> s = new ArrayList<LogicalNode>();
- s.add(node);
- while (!s.isEmpty()) {
- node = s.remove(s.size()-1);
- if (node instanceof UnaryNode) {
- UnaryNode unary = (UnaryNode) node;
- s.add(s.size(), unary.getChild());
- } else if (node instanceof BinaryNode) {
- BinaryNode binary = (BinaryNode) node;
- s.add(s.size(), binary.getLeftChild());
- s.add(s.size(), binary.getRightChild());
- } else if (node instanceof ScanNode) {
- scan.add((ScanNode)node);
- } else if (node instanceof TableSubQueryNode) {
- s.add(((TableSubQueryNode) node).getSubQuery());
- }
- }
- }
-
- private void addDataLocation(Fragment fragment) {
- String[] hosts = fragment.getHosts();
- int[] diskIds = null;
- if (fragment instanceof FileFragment) {
- diskIds = ((FileFragment)fragment).getDiskIds();
- }
- for (int i = 0; i < hosts.length; i++) {
- dataLocations.add(new DataLocation(hosts[i], diskIds == null ? -1 : diskIds[i]));
- }
- }
-
- public void addFragment(Fragment fragment, boolean useDataLocation) {
- Set<FragmentProto> fragmentProtos;
- if (fragMap.containsKey(fragment.getTableName())) {
- fragmentProtos = fragMap.get(fragment.getTableName());
- } else {
- fragmentProtos = new HashSet<FragmentProto>();
- fragMap.put(fragment.getTableName(), fragmentProtos);
- }
- fragmentProtos.add(fragment.getProto());
- if (useDataLocation) {
- addDataLocation(fragment);
- }
- totalFragmentNum++;
- }
-
- public void addFragments(Collection<Fragment> fragments) {
- for (Fragment eachFragment: fragments) {
- addFragment(eachFragment, false);
- }
- }
-
- public void setFragment(FragmentPair[] fragmentPairs) {
- for (FragmentPair eachFragmentPair : fragmentPairs) {
- this.addFragment(eachFragmentPair.getLeftFragment(), true);
- if (eachFragmentPair.getRightFragment() != null) {
- this.addFragment(eachFragmentPair.getRightFragment(), true);
- }
- }
- }
-
- public List<DataLocation> getDataLocations() {
- return dataLocations;
- }
-
- public String getSucceededHost() {
- return succeededHost;
- }
-
- public void addFetches(String tableId, Collection<FetchImpl> fetches) {
- Set<FetchImpl> fetchSet;
- if (fetchMap.containsKey(tableId)) {
- fetchSet = fetchMap.get(tableId);
- } else {
- fetchSet = Sets.newHashSet();
- }
- fetchSet.addAll(fetches);
- fetchMap.put(tableId, fetchSet);
- }
-
- public void setFetches(Map<String, Set<FetchImpl>> fetches) {
- this.fetchMap.clear();
- this.fetchMap.putAll(fetches);
- }
-
- public Collection<FragmentProto> getAllFragments() {
- Set<FragmentProto> fragmentProtos = new HashSet<FragmentProto>();
- for (Set<FragmentProto> eachFragmentSet : fragMap.values()) {
- fragmentProtos.addAll(eachFragmentSet);
- }
- return fragmentProtos;
- }
-
- public LogicalNode getLogicalPlan() {
- return this.plan;
- }
-
- public QueryUnitId getId() {
- return taskId;
- }
-
- public Collection<FetchImpl> getFetchHosts(String tableId) {
- return fetchMap.get(tableId);
- }
-
- public Collection<Set<FetchImpl>> getFetches() {
- return fetchMap.values();
- }
-
- public Map<String, Set<FetchImpl>> getFetchMap() {
- return fetchMap;
- }
-
- public Collection<FetchImpl> getFetch(ScanNode scan) {
- return this.fetchMap.get(scan.getTableName());
- }
-
- public ScanNode[] getScanNodes() {
- return this.scan.toArray(new ScanNode[scan.size()]);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append(plan.getType() + " \n");
- for (Entry<String, Set<FragmentProto>> e : fragMap.entrySet()) {
- builder.append(e.getKey()).append(" : ");
- for (FragmentProto fragment : e.getValue()) {
- builder.append(fragment).append(", ");
- }
- }
- for (Entry<String, Set<FetchImpl>> e : fetchMap.entrySet()) {
- builder.append(e.getKey()).append(" : ");
- for (FetchImpl t : e.getValue()) {
- for (URI uri : t.getURIs()){
- builder.append(uri).append(" ");
- }
- }
- }
-
- return builder.toString();
- }
-
- public void setStats(TableStats stats) {
- this.stats = stats;
- }
-
- public void setShuffleFileOutputs(List<ShuffleFileOutput> partitions) {
- this.shuffleFileOutputs = Collections.unmodifiableList(partitions);
- }
-
- public TableStats getStats() {
- return this.stats;
- }
-
- public List<ShuffleFileOutput> getShuffleFileOutputs() {
- return this.shuffleFileOutputs;
- }
-
- public int getShuffleOutpuNum() {
- return this.shuffleFileOutputs.size();
- }
-
- public QueryUnitAttempt newAttempt() {
- QueryUnitAttempt attempt = new QueryUnitAttempt(scheduleContext,
- QueryIdFactory.newQueryUnitAttemptId(this.getId(), ++nextAttempt),
- this, eventHandler);
- lastAttemptId = attempt.getId();
- return attempt;
- }
-
- public QueryUnitAttempt getAttempt(QueryUnitAttemptId attemptId) {
- return attempts.get(attemptId);
- }
-
- public QueryUnitAttempt getAttempt(int attempt) {
- return this.attempts.get(QueryIdFactory.newQueryUnitAttemptId(this.getId(), attempt));
- }
-
- public QueryUnitAttempt getLastAttempt() {
- return getAttempt(this.lastAttemptId);
- }
-
- public QueryUnitAttempt getSuccessfulAttempt() {
- readLock.lock();
- try {
- if (null == successfulAttempt) {
- return null;
- }
- return attempts.get(successfulAttempt);
- } finally {
- readLock.unlock();
- }
- }
-
- public int getRetryCount () {
- return this.nextAttempt;
- }
-
- public int getTotalFragmentNum() {
- return totalFragmentNum;
- }
-
- private static class InitialScheduleTransition implements
- SingleArcTransition<QueryUnit, TaskEvent> {
-
- @Override
- public void transition(QueryUnit task, TaskEvent taskEvent) {
- task.addAndScheduleAttempt();
- }
- }
-
- public long getLaunchTime() {
- return launchTime;
- }
-
- public long getFinishTime() {
- return finishTime;
- }
-
- @VisibleForTesting
- public void setLaunchTime(long launchTime) {
- this.launchTime = launchTime;
- }
-
- @VisibleForTesting
- public void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
- }
-
- public long getRunningTime() {
- if(finishTime > 0) {
- return finishTime - launchTime;
- } else {
- return System.currentTimeMillis() - launchTime;
- }
- }
-
- // This is always called in the Write Lock
- private void addAndScheduleAttempt() {
- // Create new task attempt
- QueryUnitAttempt attempt = newAttempt();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created attempt " + attempt.getId());
- }
- switch (attempts.size()) {
- case 0:
- attempts = Collections.singletonMap(attempt.getId(), attempt);
- break;
-
- case 1:
- Map<QueryUnitAttemptId, QueryUnitAttempt> newAttempts
- = new LinkedHashMap<QueryUnitAttemptId, QueryUnitAttempt>(3);
- newAttempts.putAll(attempts);
- attempts = newAttempts;
- attempts.put(attempt.getId(), attempt);
- break;
-
- default:
- attempts.put(attempt.getId(), attempt);
- break;
- }
-
- if (failedAttempts > 0) {
- eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
- TaskAttemptEventType.TA_RESCHEDULE));
- } else {
- eventHandler.handle(new TaskAttemptScheduleEvent(systemConf, attempt.getId(),
- TaskAttemptEventType.TA_SCHEDULE));
- }
- }
-
- private void finishTask() {
- this.finishTime = System.currentTimeMillis();
- finalQueryUnitHistory = makeQueryUnitHistory();
- }
-
- private static class KillNewTaskTransition implements SingleArcTransition<QueryUnit, TaskEvent> {
-
- @Override
- public void transition(QueryUnit task, TaskEvent taskEvent) {
- task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.KILLED));
- }
- }
-
- private static class KillTaskTransition implements SingleArcTransition<QueryUnit, TaskEvent> {
-
- @Override
- public void transition(QueryUnit task, TaskEvent taskEvent) {
- task.finishTask();
- task.eventHandler.handle(new TaskAttemptEvent(task.lastAttemptId, TaskAttemptEventType.TA_KILL));
- }
- }
-
- private static class AttemptKilledTransition implements SingleArcTransition<QueryUnit, TaskEvent>{
-
- @Override
- public void transition(QueryUnit task, TaskEvent event) {
- task.finishTask();
- task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.KILLED));
- }
- }
-
- private static class AttemptSucceededTransition
- implements SingleArcTransition<QueryUnit, TaskEvent>{
-
- @Override
- public void transition(QueryUnit task,
- TaskEvent event) {
- TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
- QueryUnitAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
-
- task.successfulAttempt = attemptEvent.getTaskAttemptId();
- task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
- task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
- task.succeededPullServerPort = attempt.getWorkerConnectionInfo().getPullServerPort();
-
- task.finishTask();
- task.eventHandler.handle(new SubQueryTaskEvent(event.getTaskId(), TaskState.SUCCEEDED));
- }
- }
-
- private static class AttemptLaunchedTransition implements SingleArcTransition<QueryUnit, TaskEvent> {
- @Override
- public void transition(QueryUnit task,
- TaskEvent event) {
- TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
- QueryUnitAttempt attempt = task.attempts.get(attemptEvent.getTaskAttemptId());
- task.launchTime = System.currentTimeMillis();
- task.succeededHost = attempt.getWorkerConnectionInfo().getHost();
- task.succeededHostPort = attempt.getWorkerConnectionInfo().getPeerRpcPort();
- }
- }
-
- private static class AttemptFailedTransition implements SingleArcTransition<QueryUnit, TaskEvent> {
- @Override
- public void transition(QueryUnit task, TaskEvent event) {
- TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) event;
- LOG.info("=============================================================");
- LOG.info(">>> Task Failed: " + attemptEvent.getTaskAttemptId() + " <<<");
- LOG.info("=============================================================");
- task.failedAttempts++;
- task.finishedAttempts++;
-
- task.finishTask();
- task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.FAILED));
- }
- }
-
- private static class AttemptFailedOrRetryTransition implements
- MultipleArcTransition<QueryUnit, TaskEvent, TaskState> {
-
- @Override
- public TaskState transition(QueryUnit task, TaskEvent taskEvent) {
- TaskTAttemptEvent attemptEvent = (TaskTAttemptEvent) taskEvent;
- task.failedAttempts++;
- task.finishedAttempts++;
- boolean retry = task.failedAttempts < task.maxAttempts;
-
- LOG.info("====================================================================================");
- LOG.info(">>> Task Failed: " + attemptEvent.getTaskAttemptId() + ", " +
- "retry:" + retry + ", attempts:" + task.failedAttempts + " <<<");
- LOG.info("====================================================================================");
-
- if (retry) {
- if (task.successfulAttempt == null) {
- task.addAndScheduleAttempt();
- }
- } else {
- task.finishTask();
- task.eventHandler.handle(new SubQueryTaskEvent(task.getId(), TaskState.FAILED));
- return TaskState.FAILED;
- }
-
- return task.getState();
- }
- }
-
- @Override
- public void handle(TaskEvent event) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing " + event.getTaskId() + " of type "
- + event.getType());
- }
-
- try {
- writeLock.lock();
- TaskState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitonException e) {
- LOG.error("Can't handle this event at current state"
- + ", eventType:" + event.getType().name()
- + ", oldState:" + oldState.name()
- + ", nextState:" + getState().name()
- , e);
- eventHandler.handle(new QueryEvent(TajoIdUtils.parseQueryId(getId().toString()),
- QueryEventType.INTERNAL_ERROR));
- }
-
- //notify the eventhandler of state change
- if (LOG.isDebugEnabled()) {
- if (oldState != getState()) {
- LOG.debug(taskId + " Task Transitioned from " + oldState + " to "
- + getState());
- }
- }
- }
-
- finally {
- writeLock.unlock();
- }
- }
-
- public void setIntermediateData(Collection<IntermediateEntry> partitions) {
- this.intermediateData = new ArrayList<IntermediateEntry>(partitions);
- }
-
- public List<IntermediateEntry> getIntermediateData() {
- return this.intermediateData;
- }
-
- public static class PullHost implements Cloneable {
- String host;
- int port;
- int hashCode;
-
- public PullHost(String pullServerAddr, int pullServerPort){
- this.host = pullServerAddr;
- this.port = pullServerPort;
- this.hashCode = Objects.hashCode(host, port);
- }
- public String getHost() {
- return host;
- }
-
- public int getPort() {
- return this.port;
- }
-
- public String getPullAddress() {
- return host + ":" + port;
- }
-
- @Override
- public int hashCode() {
- return hashCode;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof PullHost) {
- PullHost other = (PullHost) obj;
- return host.equals(other.host) && port == other.port;
- }
-
- return false;
- }
-
- @Override
- public PullHost clone() throws CloneNotSupportedException {
- PullHost newPullHost = (PullHost) super.clone();
- newPullHost.host = host;
- newPullHost.port = port;
- newPullHost.hashCode = Objects.hashCode(newPullHost.host, newPullHost.port);
- return newPullHost;
- }
-
- @Override
- public String toString() {
- return host + ":" + port;
- }
- }
-
- public static class IntermediateEntry {
- ExecutionBlockId ebId;
- int taskId;
- int attemptId;
- int partId;
- PullHost host;
- long volume;
- List<Pair<Long, Integer>> pages;
- List<Pair<Long, Pair<Integer, Integer>>> failureRowNums;
-
- public IntermediateEntry(IntermediateEntryProto proto) {
- this.ebId = new ExecutionBlockId(proto.getEbId());
- this.taskId = proto.getTaskId();
- this.attemptId = proto.getAttemptId();
- this.partId = proto.getPartId();
-
- String[] pullHost = proto.getHost().split(":");
- this.host = new PullHost(pullHost[0], Integer.parseInt(pullHost[1]));
- this.volume = proto.getVolume();
-
- failureRowNums = new ArrayList<Pair<Long, Pair<Integer, Integer>>>();
- for (FailureIntermediateProto eachFailure: proto.getFailuresList()) {
-
- failureRowNums.add(new Pair(eachFailure.getPagePos(),
- new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum())));
- }
-
- pages = new ArrayList<Pair<Long, Integer>>();
- for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) {
- pages.add(new Pair(eachPage.getPos(), eachPage.getLength()));
- }
- }
-
- public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host) {
- this.taskId = taskId;
- this.attemptId = attemptId;
- this.partId = partId;
- this.host = host;
- }
-
- public IntermediateEntry(int taskId, int attemptId, int partId, PullHost host, long volume) {
- this.taskId = taskId;
- this.attemptId = attemptId;
- this.partId = partId;
- this.host = host;
- this.volume = volume;
- }
-
- public ExecutionBlockId getEbId() {
- return ebId;
- }
-
- public void setEbId(ExecutionBlockId ebId) {
- this.ebId = ebId;
- }
-
- public int getTaskId() {
- return this.taskId;
- }
-
- public int getAttemptId() {
- return this.attemptId;
- }
-
- public int getPartId() {
- return this.partId;
- }
-
- public PullHost getPullHost() {
- return this.host;
- }
-
- public long getVolume() {
- return this.volume;
- }
-
- public long setVolume(long volume) {
- return this.volume = volume;
- }
-
- public List<Pair<Long, Integer>> getPages() {
- return pages;
- }
-
- public void setPages(List<Pair<Long, Integer>> pages) {
- this.pages = pages;
- }
-
- public List<Pair<Long, Pair<Integer, Integer>>> getFailureRowNums() {
- return failureRowNums;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(ebId, taskId, partId, attemptId, host);
- }
-
- public List<Pair<Long, Long>> split(long firstSplitVolume, long splitVolume) {
- List<Pair<Long, Long>> splits = new ArrayList<Pair<Long, Long>>();
-
- if (pages == null || pages.isEmpty()) {
- return splits;
- }
- int pageSize = pages.size();
-
- long currentOffset = -1;
- long currentBytes = 0;
-
- long realSplitVolume = firstSplitVolume > 0 ? firstSplitVolume : splitVolume;
- for (int i = 0; i < pageSize; i++) {
- Pair<Long, Integer> eachPage = pages.get(i);
- if (currentOffset == -1) {
- currentOffset = eachPage.getFirst();
- }
- if (currentBytes > 0 && currentBytes + eachPage.getSecond() >= realSplitVolume) {
- splits.add(new Pair(currentOffset, currentBytes));
- currentOffset = eachPage.getFirst();
- currentBytes = 0;
- realSplitVolume = splitVolume;
- }
-
- currentBytes += eachPage.getSecond();
- }
-
- //add last
- if (currentBytes > 0) {
- splits.add(new Pair(currentOffset, currentBytes));
- }
- return splits;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
deleted file mode 100644
index d88173f..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.querymaster;
-
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.state.*;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.TajoProtos.TaskAttemptState;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.ipc.TajoWorkerProtocol.TaskCompletionReport;
-import org.apache.tajo.master.cluster.WorkerConnectionInfo;
-import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
-import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
-import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
-import org.apache.tajo.master.querymaster.QueryUnit.PullHost;
-import org.apache.tajo.master.container.TajoContainerId;
-
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleFileOutput;
-
-public class QueryUnitAttempt implements EventHandler<TaskAttemptEvent> {
-
- private static final Log LOG = LogFactory.getLog(QueryUnitAttempt.class);
-
- private final static int EXPIRE_TIME = 15000;
-
- private final QueryUnitAttemptId id;
- private final QueryUnit queryUnit;
- final EventHandler eventHandler;
-
- private TajoContainerId containerId;
- private WorkerConnectionInfo workerConnectionInfo;
- private int expire;
-
- private final Lock readLock;
- private final Lock writeLock;
-
- private final List<String> diagnostics = new ArrayList<String>();
-
- private final QueryUnitAttemptScheduleContext scheduleContext;
-
- private float progress;
- private CatalogProtos.TableStatsProto inputStats;
- private CatalogProtos.TableStatsProto resultStats;
-
- protected static final StateMachineFactory
- <QueryUnitAttempt, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
- stateMachineFactory = new StateMachineFactory
- <QueryUnitAttempt, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
- (TaskAttemptState.TA_NEW)
-
- // Transitions from TA_NEW state
- .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_UNASSIGNED,
- TaskAttemptEventType.TA_SCHEDULE, new TaskAttemptScheduleTransition())
- .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_UNASSIGNED,
- TaskAttemptEventType.TA_RESCHEDULE, new TaskAttemptScheduleTransition())
- .addTransition(TaskAttemptState.TA_NEW, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_KILL,
- new TaskKilledCompleteTransition())
-
- // Transitions from TA_UNASSIGNED state
- .addTransition(TaskAttemptState.TA_UNASSIGNED, TaskAttemptState.TA_ASSIGNED,
- TaskAttemptEventType.TA_ASSIGNED,
- new LaunchTransition())
- .addTransition(TaskAttemptState.TA_UNASSIGNED, TaskAttemptState.TA_KILL_WAIT,
- TaskAttemptEventType.TA_KILL,
- new KillUnassignedTaskTransition())
-
- // Transitions from TA_ASSIGNED state
- .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_ASSIGNED,
- TaskAttemptEventType.TA_ASSIGNED, new AlreadyAssignedTransition())
- .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_KILL_WAIT,
- TaskAttemptEventType.TA_KILL,
- new KillTaskTransition())
- .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_KILL,
- new KillTaskTransition())
- .addTransition(TaskAttemptState.TA_ASSIGNED,
- EnumSet.of(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_KILLED),
- TaskAttemptEventType.TA_UPDATE, new StatusUpdateTransition())
- .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_SUCCEEDED,
- TaskAttemptEventType.TA_DONE, new SucceededTransition())
- .addTransition(TaskAttemptState.TA_ASSIGNED, TaskAttemptState.TA_FAILED,
- TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
-
- // Transitions from TA_RUNNING state
- .addTransition(TaskAttemptState.TA_RUNNING,
- EnumSet.of(TaskAttemptState.TA_RUNNING),
- TaskAttemptEventType.TA_UPDATE, new StatusUpdateTransition())
- .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_KILL_WAIT,
- TaskAttemptEventType.TA_KILL,
- new KillTaskTransition())
- .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_SUCCEEDED,
- TaskAttemptEventType.TA_DONE, new SucceededTransition())
- .addTransition(TaskAttemptState.TA_RUNNING, TaskAttemptState.TA_FAILED,
- TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
-
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_LOCAL_KILLED,
- new TaskKilledCompleteTransition())
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILL_WAIT,
- TaskAttemptEventType.TA_ASSIGNED,
- new KillTaskTransition())
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_SCHEDULE_CANCELED,
- new TaskKilledCompleteTransition())
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_DONE,
- new TaskKilledCompleteTransition())
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_FAILED,
- TaskAttemptEventType.TA_FATAL_ERROR)
- .addTransition(TaskAttemptState.TA_KILL_WAIT, TaskAttemptState.TA_KILL_WAIT,
- EnumSet.of(
- TaskAttemptEventType.TA_KILL,
- TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
- TaskAttemptEventType.TA_UPDATE))
-
- // Transitions from TA_SUCCEEDED state
- .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
- TaskAttemptEventType.TA_UPDATE)
- .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
- TaskAttemptEventType.TA_DONE, new AlreadyDoneTransition())
- .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_FAILED,
- TaskAttemptEventType.TA_FATAL_ERROR, new FailedTransition())
- // Ignore-able transitions
- .addTransition(TaskAttemptState.TA_SUCCEEDED, TaskAttemptState.TA_SUCCEEDED,
- TaskAttemptEventType.TA_KILL)
-
- // Transitions from TA_KILLED state
- .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
- TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE)
- // Ignore-able transitions
- .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
- EnumSet.of(
- TaskAttemptEventType.TA_UPDATE))
- .addTransition(TaskAttemptState.TA_KILLED, TaskAttemptState.TA_KILLED,
- EnumSet.of(
- TaskAttemptEventType.TA_LOCAL_KILLED,
- TaskAttemptEventType.TA_KILL,
- TaskAttemptEventType.TA_ASSIGNED,
- TaskAttemptEventType.TA_DONE),
- new TaskKilledCompleteTransition())
- .installTopology();
-
- private final StateMachine<TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
- stateMachine;
-
-
- public QueryUnitAttempt(final QueryUnitAttemptScheduleContext scheduleContext,
- final QueryUnitAttemptId id, final QueryUnit queryUnit,
- final EventHandler eventHandler) {
- this.scheduleContext = scheduleContext;
- this.id = id;
- this.expire = QueryUnitAttempt.EXPIRE_TIME;
- this.queryUnit = queryUnit;
- this.eventHandler = eventHandler;
-
- ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- this.readLock = readWriteLock.readLock();
- this.writeLock = readWriteLock.writeLock();
-
- stateMachine = stateMachineFactory.make(this);
- }
-
- public TaskAttemptState getState() {
- readLock.lock();
- try {
- return stateMachine.getCurrentState();
- } finally {
- readLock.unlock();
- }
- }
-
- public QueryUnitAttemptId getId() {
- return this.id;
- }
-
- public boolean isLeafTask() {
- return this.queryUnit.isLeafTask();
- }
-
- public QueryUnit getQueryUnit() {
- return this.queryUnit;
- }
-
- public WorkerConnectionInfo getWorkerConnectionInfo() {
- return this.workerConnectionInfo;
- }
-
- public void setContainerId(TajoContainerId containerId) {
- this.containerId = containerId;
- }
-
- public synchronized void setExpireTime(int expire) {
- this.expire = expire;
- }
-
- public synchronized void updateExpireTime(int period) {
- this.setExpireTime(this.expire - period);
- }
-
- public synchronized void resetExpireTime() {
- this.setExpireTime(QueryUnitAttempt.EXPIRE_TIME);
- }
-
- public int getLeftTime() {
- return this.expire;
- }
-
- public float getProgress() {
- return progress;
- }
-
- public TableStats getInputStats() {
- if (inputStats == null) {
- return null;
- }
-
- return new TableStats(inputStats);
- }
-
- public TableStats getResultStats() {
- if (resultStats == null) {
- return null;
- }
- return new TableStats(resultStats);
- }
-
- private void fillTaskStatistics(TaskCompletionReport report) {
- this.progress = 1.0f;
-
- List<IntermediateEntry> partitions = new ArrayList<IntermediateEntry>();
-
- if (report.getShuffleFileOutputsCount() > 0) {
- this.getQueryUnit().setShuffleFileOutputs(report.getShuffleFileOutputsList());
-
- PullHost host = new PullHost(getWorkerConnectionInfo().getHost(), getWorkerConnectionInfo().getPullServerPort());
- for (ShuffleFileOutput p : report.getShuffleFileOutputsList()) {
- IntermediateEntry entry = new IntermediateEntry(getId().getQueryUnitId().getId(),
- getId().getId(), p.getPartId(), host, p.getVolume());
- partitions.add(entry);
- }
- }
- this.getQueryUnit().setIntermediateData(partitions);
-
- if (report.hasInputStats()) {
- this.inputStats = report.getInputStats();
- }
- if (report.hasResultStats()) {
- this.resultStats = report.getResultStats();
- this.getQueryUnit().setStats(new TableStats(resultStats));
- }
- }
-
- private static class TaskAttemptScheduleTransition implements
- SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
-
- @Override
- public void transition(QueryUnitAttempt taskAttempt, TaskAttemptEvent taskAttemptEvent) {
- taskAttempt.eventHandler.handle(new QueryUnitAttemptScheduleEvent(
- EventType.T_SCHEDULE, taskAttempt.getQueryUnit().getId().getExecutionBlockId(),
- taskAttempt.scheduleContext, taskAttempt));
- }
- }
-
- private static class KillUnassignedTaskTransition implements
- SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
-
- @Override
- public void transition(QueryUnitAttempt taskAttempt, TaskAttemptEvent taskAttemptEvent) {
- taskAttempt.eventHandler.handle(new QueryUnitAttemptScheduleEvent(
- EventType.T_SCHEDULE_CANCEL, taskAttempt.getQueryUnit().getId().getExecutionBlockId(),
- taskAttempt.scheduleContext, taskAttempt));
- }
- }
-
- private static class LaunchTransition
- implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
-
- @Override
- public void transition(QueryUnitAttempt taskAttempt,
- TaskAttemptEvent event) {
- TaskAttemptAssignedEvent castEvent = (TaskAttemptAssignedEvent) event;
- taskAttempt.containerId = castEvent.getContainerId();
- taskAttempt.workerConnectionInfo = castEvent.getWorkerConnectionInfo();
- taskAttempt.eventHandler.handle(
- new TaskTAttemptEvent(taskAttempt.getId(),
- TaskEventType.T_ATTEMPT_LAUNCHED));
- }
- }
-
- private static class TaskKilledCompleteTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
-
- @Override
- public void transition(QueryUnitAttempt taskAttempt,
- TaskAttemptEvent event) {
- taskAttempt.getQueryUnit().handle(new TaskEvent(taskAttempt.getId().getQueryUnitId(),
- TaskEventType.T_ATTEMPT_KILLED));
- LOG.info(taskAttempt.getId() + " Received TA_KILLED Status from LocalTask");
- }
- }
-
- private static class StatusUpdateTransition
- implements MultipleArcTransition<QueryUnitAttempt, TaskAttemptEvent, TaskAttemptState> {
-
- @Override
- public TaskAttemptState transition(QueryUnitAttempt taskAttempt,
- TaskAttemptEvent event) {
- TaskAttemptStatusUpdateEvent updateEvent = (TaskAttemptStatusUpdateEvent) event;
-
- taskAttempt.progress = updateEvent.getStatus().getProgress();
- taskAttempt.inputStats = updateEvent.getStatus().getInputStats();
- taskAttempt.resultStats = updateEvent.getStatus().getResultStats();
-
- return TaskAttemptState.TA_RUNNING;
- }
- }
-
- private void addDiagnosticInfo(String diag) {
- if (diag != null && !diag.equals("")) {
- diagnostics.add(diag);
- }
- }
-
- private static class AlreadyAssignedTransition
- implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
-
- @Override
- public void transition(QueryUnitAttempt queryUnitAttempt,
- TaskAttemptEvent taskAttemptEvent) {
- }
- }
-
- private static class AlreadyDoneTransition
- implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
-
- @Override
- public void transition(QueryUnitAttempt queryUnitAttempt,
- TaskAttemptEvent taskAttemptEvent) {
- }
- }
-
- private static class SucceededTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
- @Override
- public void transition(QueryUnitAttempt taskAttempt,
- TaskAttemptEvent event) {
- TaskCompletionReport report = ((TaskCompletionEvent)event).getReport();
-
- try {
- taskAttempt.fillTaskStatistics(report);
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
- } catch (Throwable t) {
- taskAttempt.eventHandler.handle(new TaskFatalErrorEvent(taskAttempt.getId(), t.getMessage()));
- taskAttempt.addDiagnosticInfo(ExceptionUtils.getStackTrace(t));
- }
- }
- }
-
- private static class KillTaskTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent> {
-
- @Override
- public void transition(QueryUnitAttempt taskAttempt, TaskAttemptEvent event) {
- taskAttempt.eventHandler.handle(new LocalTaskEvent(taskAttempt.getId(), taskAttempt.containerId,
- LocalTaskEventType.KILL));
- }
- }
-
- private static class FailedTransition implements SingleArcTransition<QueryUnitAttempt, TaskAttemptEvent>{
- @Override
- public void transition(QueryUnitAttempt taskAttempt, TaskAttemptEvent event) {
- TaskFatalErrorEvent errorEvent = (TaskFatalErrorEvent) event;
- taskAttempt.eventHandler.handle(new TaskTAttemptEvent(taskAttempt.getId(), TaskEventType.T_ATTEMPT_FAILED));
- taskAttempt.addDiagnosticInfo(errorEvent.errorMessage());
- LOG.error(taskAttempt.getId() + " FROM " + taskAttempt.getWorkerConnectionInfo().getHost()
- + " >> " + errorEvent.errorMessage());
- }
- }
-
- @Override
- public void handle(TaskAttemptEvent event) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing " + event.getTaskAttemptId() + " of type " + event.getType());
- }
- try {
- writeLock.lock();
- TaskAttemptState oldState = getState();
- try {
- stateMachine.doTransition(event.getType(), event);
- } catch (InvalidStateTransitonException e) {
- LOG.error("Can't handle this event at current state of " + event.getTaskAttemptId() + ")"
- + ", eventType:" + event.getType().name()
- + ", oldState:" + oldState.name()
- + ", nextState:" + getState().name()
- , e);
- eventHandler.handle(
- new SubQueryDiagnosticsUpdateEvent(event.getTaskAttemptId().getQueryUnitId().getExecutionBlockId(),
- "Can't handle this event at current state of " + event.getTaskAttemptId() + ")"));
- eventHandler.handle(
- new SubQueryEvent(event.getTaskAttemptId().getQueryUnitId().getExecutionBlockId(),
- SubQueryEventType.SQ_INTERNAL_ERROR));
- }
-
- //notify the eventhandler of state change
- if (LOG.isDebugEnabled()) {
- if (oldState != getState()) {
- LOG.debug(id + " TaskAttempt Transitioned from " + oldState + " to "
- + getState());
- }
- }
- }
-
- finally {
- writeLock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index a240ace..cf6b917 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -44,7 +44,7 @@ import org.apache.tajo.exception.InternalException;
import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage;
import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty;
import org.apache.tajo.master.TaskSchedulerContext;
-import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
+import org.apache.tajo.master.querymaster.Task.IntermediateEntry;
import org.apache.tajo.plan.logical.SortNode.SortPurpose;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.PlanningException;
@@ -717,7 +717,7 @@ public class Repartitioner {
List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
for (ExecutionBlock childBlock : childBlocks) {
SubQuery childExecSM = subQuery.getContext().getSubQuery(childBlock.getId());
- for (QueryUnit qu : childExecSM.getQueryUnits()) {
+ for (Task qu : childExecSM.getTasks()) {
for (IntermediateEntry p : qu.getIntermediateData()) {
FetchImpl fetch = new FetchImpl(p.getPullHost(), RANGE_SHUFFLE, childBlock.getId(), 0);
fetch.addPart(p.getTaskId(), p.getAttemptId());
@@ -840,8 +840,8 @@ public class Repartitioner {
// make FetchImpl per PullServer, PartId
Map<Integer, List<IntermediateEntry>> hashed = hashByKey(partitions);
for (Entry<Integer, List<IntermediateEntry>> interm : hashed.entrySet()) {
- Map<QueryUnit.PullHost, List<IntermediateEntry>> hashedByHost = hashByHost(interm.getValue());
- for (Entry<QueryUnit.PullHost, List<IntermediateEntry>> e : hashedByHost.entrySet()) {
+ Map<Task.PullHost, List<IntermediateEntry>> hashedByHost = hashByHost(interm.getValue());
+ for (Entry<Task.PullHost, List<IntermediateEntry>> e : hashedByHost.entrySet()) {
FetchImpl fetch = new FetchImpl(e.getKey(), channel.getShuffleType(),
block.getId(), interm.getKey(), e.getValue());
@@ -1191,10 +1191,10 @@ public class Repartitioner {
return hashed;
}
- public static Map<QueryUnit.PullHost, List<IntermediateEntry>> hashByHost(List<IntermediateEntry> entries) {
- Map<QueryUnit.PullHost, List<IntermediateEntry>> hashed = new HashMap<QueryUnit.PullHost, List<IntermediateEntry>>();
+ public static Map<Task.PullHost, List<IntermediateEntry>> hashByHost(List<IntermediateEntry> entries) {
+ Map<Task.PullHost, List<IntermediateEntry>> hashed = new HashMap<Task.PullHost, List<IntermediateEntry>>();
- QueryUnit.PullHost host;
+ Task.PullHost host;
for (IntermediateEntry entry : entries) {
host = entry.getPullHost();
if (hashed.containsKey(host)) {
[03/13] tajo git commit: TAJO-1254: Fix getProgress race conditions
in Query. (jinho)
Posted by ji...@apache.org.
TAJO-1254: Fix getProgress race conditions in Query. (jinho)
Closes #304
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ff57c77e
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ff57c77e
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ff57c77e
Branch: refs/heads/index_support
Commit: ff57c77ea6ea790a0689b59272600325e5b73345
Parents: 2548768
Author: jhkim <jh...@apache.org>
Authored: Thu Dec 18 11:57:43 2014 +0900
Committer: jhkim <jh...@apache.org>
Committed: Thu Dec 18 11:57:43 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../src/main/java/org/apache/tajo/master/querymaster/Query.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ff57c77e/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index de63285..6b9146a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -111,6 +111,8 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1254: Fix getProgress race conditions in Query. (jinho)
+
TAJO-1252: PathValidator should allow hdfs paths which contain IP addresses.
(jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/ff57c77e/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
index a048780..918cc82 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -206,7 +206,7 @@ public class Query implements EventHandler<QueryEvent> {
this.clock = context.getClock();
this.appSubmitTime = appSubmitTime;
this.queryStr = queryStr;
- subqueries = Maps.newHashMap();
+ this.subqueries = Maps.newConcurrentMap();
this.eventHandler = eventHandler;
this.plan = plan;
this.cursor = new ExecutionBlockCursor(plan, true);
[06/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
index f9c752b..208591f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java
@@ -24,9 +24,8 @@ import com.google.common.collect.Lists;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.querymaster.QueryUnit;
+import org.apache.tajo.master.querymaster.Task;
import org.apache.tajo.master.querymaster.Repartitioner;
-import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.util.TUtil;
import java.net.URI;
@@ -39,7 +38,7 @@ import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
* <code>FetchImpl</code> information to indicate the locations of intermediate data.
*/
public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cloneable {
- private QueryUnit.PullHost host; // The pull server host information
+ private Task.PullHost host; // The pull server host information
private ShuffleType type; // hash or range partition method.
private ExecutionBlockId executionBlockId; // The executionBlock id
private int partitionId; // The hash partition id
@@ -59,7 +58,7 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
}
public FetchImpl(TajoWorkerProtocol.FetchProto proto) {
- this(new QueryUnit.PullHost(proto.getHost(), proto.getPort()),
+ this(new Task.PullHost(proto.getHost(), proto.getPort()),
proto.getType(),
new ExecutionBlockId(proto.getExecutionBlockId()),
proto.getPartitionId(),
@@ -77,22 +76,22 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
}
}
- public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
+ public FetchImpl(Task.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
int partitionId) {
this(host, type, executionBlockId, partitionId, null, false, null,
new ArrayList<Integer>(), new ArrayList<Integer>());
}
- public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
- int partitionId, List<QueryUnit.IntermediateEntry> intermediateEntryList) {
+ public FetchImpl(Task.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
+ int partitionId, List<Task.IntermediateEntry> intermediateEntryList) {
this(host, type, executionBlockId, partitionId, null, false, null,
new ArrayList<Integer>(), new ArrayList<Integer>());
- for (QueryUnit.IntermediateEntry entry : intermediateEntryList){
+ for (Task.IntermediateEntry entry : intermediateEntryList){
addPart(entry.getTaskId(), entry.getAttemptId());
}
}
- public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
+ public FetchImpl(Task.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId,
int partitionId, String rangeParams, boolean hasNext, String name,
List<Integer> taskIds, List<Integer> attemptIds) {
this.host = host;
@@ -142,7 +141,7 @@ public class FetchImpl implements ProtoObject<TajoWorkerProtocol.FetchProto>, Cl
this.attemptIds.add(attemptId);
}
- public QueryUnit.PullHost getPullHost() {
+ public Task.PullHost getPullHost() {
return this.host;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/InterDataRetriever.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/InterDataRetriever.java b/tajo-core/src/main/java/org/apache/tajo/worker/InterDataRetriever.java
index 42ad875..5b2ad0f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/InterDataRetriever.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/InterDataRetriever.java
@@ -27,7 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.handler.codec.http.HttpRequest;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.worker.dataserver.FileAccessForbiddenException;
import org.apache.tajo.worker.dataserver.retriever.DataRetriever;
import org.apache.tajo.worker.dataserver.retriever.FileChunk;
@@ -41,13 +41,13 @@ import java.util.Set;
@Deprecated
public class InterDataRetriever implements DataRetriever {
private final Log LOG = LogFactory.getLog(InterDataRetriever.class);
- private final Set<QueryUnitId> registered = Sets.newHashSet();
+ private final Set<TaskId> registered = Sets.newHashSet();
private final Map<String, String> map = Maps.newConcurrentMap();
public InterDataRetriever() {
}
- public void register(QueryUnitId id, String baseURI) {
+ public void register(TaskId id, String baseURI) {
synchronized (registered) {
if (!registered.contains(id)) {
map.put(id.toString(), baseURI);
@@ -56,7 +56,7 @@ public class InterDataRetriever implements DataRetriever {
}
}
- public void unregister(QueryUnitId id) {
+ public void unregister(TaskId id) {
synchronized (registered) {
if (registered.contains(id)) {
map.remove(id.toString());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
index 48f4f66..4a09772 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
@@ -104,7 +104,7 @@ public class TajoWorkerManagerService extends CompositeService
@Override
public void ping(RpcController controller,
- TajoIdProtos.QueryUnitAttemptIdProto attemptId,
+ TajoIdProtos.TaskAttemptIdProto attemptId,
RpcCallback<PrimitiveProtos.BoolProto> done) {
done.run(TajoWorker.TRUE_PROTO);
}
@@ -146,9 +146,9 @@ public class TajoWorkerManagerService extends CompositeService
}
@Override
- public void killTaskAttempt(RpcController controller, TajoIdProtos.QueryUnitAttemptIdProto request,
+ public void killTaskAttempt(RpcController controller, TajoIdProtos.TaskAttemptIdProto request,
RpcCallback<PrimitiveProtos.BoolProto> done) {
- Task task = workerContext.getTaskRunnerManager().getTaskByQueryUnitAttemptId(new QueryUnitAttemptId(request));
+ Task task = workerContext.getTaskRunnerManager().getTaskByTaskAttemptId(new TaskAttemptId(request));
if(task != null) task.kill();
done.run(TajoWorker.TRUE_PROTO);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 00eabcc..0920619 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.TajoProtos.TaskAttemptState;
import org.apache.tajo.catalog.Schema;
@@ -44,7 +44,7 @@ import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.engine.planner.physical.PhysicalExec;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.engine.query.QueryUnitRequest;
+import org.apache.tajo.engine.query.TaskRequest;
import org.apache.tajo.ipc.QueryMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol.*;
import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType;
@@ -63,7 +63,6 @@ import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
-import java.text.NumberFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ExecutorService;
@@ -78,11 +77,11 @@ public class Task {
private final TajoConf systemConf;
private final QueryContext queryContext;
private final ExecutionBlockContext executionBlockContext;
- private final QueryUnitAttemptId taskId;
+ private final TaskAttemptId taskId;
private final String taskRunnerId;
private final Path taskDir;
- private final QueryUnitRequest request;
+ private final TaskRequest request;
private TaskAttemptContext context;
private List<Fetcher> fetcherRunners;
private LogicalNode plan;
@@ -106,9 +105,9 @@ public class Task {
public Task(String taskRunnerId,
Path baseDir,
- QueryUnitAttemptId taskId,
+ TaskAttemptId taskId,
final ExecutionBlockContext executionBlockContext,
- final QueryUnitRequest request) throws IOException {
+ final TaskRequest request) throws IOException {
this.taskRunnerId = taskRunnerId;
this.request = request;
this.taskId = taskId;
@@ -117,7 +116,7 @@ public class Task {
this.queryContext = request.getQueryContext(systemConf);
this.executionBlockContext = executionBlockContext;
this.taskDir = StorageUtil.concatPath(baseDir,
- taskId.getQueryUnitId().getId() + "_" + taskId.getId());
+ taskId.getTaskId().getId() + "_" + taskId.getId());
this.context = new TaskAttemptContext(queryContext, executionBlockContext, taskId,
request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir);
@@ -207,7 +206,7 @@ public class Task {
}
}
- public QueryUnitAttemptId getTaskId() {
+ public TaskAttemptId getTaskId() {
return taskId;
}
@@ -215,11 +214,11 @@ public class Task {
return LOG;
}
- public void localize(QueryUnitRequest request) throws IOException {
+ public void localize(TaskRequest request) throws IOException {
fetcherRunners = getFetchRunners(context, request.getFetches());
}
- public QueryUnitAttemptId getId() {
+ public TaskAttemptId getId() {
return context.getTaskId();
}
@@ -272,7 +271,7 @@ public class Task {
executionBlockContext.getTasks().remove(this.getId());
}
} else {
- LOG.error("QueryUnitAttemptId: " + context.getTaskId() + " status: " + context.getState());
+ LOG.error("TaskAttemptId: " + context.getTaskId() + " status: " + context.getState());
}
}
@@ -626,7 +625,7 @@ public class Task {
if (retryNum == maxRetryNum) {
LOG.error("ERROR: the maximum retry (" + retryNum + ") on the fetch exceeded (" + fetcher.getURI() + ")");
}
- aborted = true; // retry queryUnit
+ aborted = true; // retry task
ctx.getFetchLatch().countDown();
}
}
@@ -830,10 +829,10 @@ public class Task {
return ret;
}
- public static Path getTaskAttemptDir(QueryUnitAttemptId quid) {
+ public static Path getTaskAttemptDir(TaskAttemptId quid) {
Path workDir =
- StorageUtil.concatPath(ExecutionBlockContext.getBaseInputDir(quid.getQueryUnitId().getExecutionBlockId()),
- String.valueOf(quid.getQueryUnitId().getId()),
+ StorageUtil.concatPath(ExecutionBlockContext.getBaseInputDir(quid.getTaskId().getExecutionBlockId()),
+ String.valueOf(quid.getTaskId().getId()),
String.valueOf(quid.getId()));
return workDir;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index 99976d8..1556a44 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -24,7 +24,7 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoProtos.TaskAttemptState;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -59,7 +59,7 @@ public class TaskAttemptContext {
private TaskAttemptState state;
private TableStats resultStats;
- private QueryUnitAttemptId queryId;
+ private TaskAttemptId queryId;
private final Path workDir;
private boolean needFetch = false;
private CountDownLatch doneFetchPhaseSignal;
@@ -84,7 +84,7 @@ public class TaskAttemptContext {
private HashShuffleAppenderManager hashShuffleAppenderManager;
public TaskAttemptContext(QueryContext queryContext, final ExecutionBlockContext executionBlockContext,
- final QueryUnitAttemptId queryId,
+ final TaskAttemptId queryId,
final FragmentProto[] fragments,
final Path workDir) {
this.queryContext = queryContext;
@@ -127,7 +127,7 @@ public class TaskAttemptContext {
}
@VisibleForTesting
- public TaskAttemptContext(final QueryContext queryContext, final QueryUnitAttemptId queryId,
+ public TaskAttemptContext(final QueryContext queryContext, final TaskAttemptId queryId,
final Fragment [] fragments, final Path workDir) {
this(queryContext, null, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir);
}
@@ -306,7 +306,7 @@ public class TaskAttemptContext {
return this.workDir;
}
- public QueryUnitAttemptId getTaskId() {
+ public TaskAttemptId getTaskId() {
return this.queryId;
}
@@ -396,7 +396,7 @@ public class TaskAttemptContext {
return queryContext;
}
- public QueryUnitAttemptId getQueryId() {
+ public TaskAttemptId getQueryId() {
return queryId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
index 9b6fd0d..c2432eb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskHistory.java
@@ -20,7 +20,7 @@ package org.apache.tajo.worker;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.util.history.History;
@@ -37,7 +37,7 @@ import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskHistoryProto;
*/
public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
- private QueryUnitAttemptId queryUnitAttemptId;
+ private TaskAttemptId taskAttemptId;
private TaskAttemptState state;
private float progress;
private long startTime;
@@ -51,10 +51,10 @@ public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
private int totalFetchCount;
private List<FetcherHistoryProto> fetcherHistories;
- public TaskHistory(QueryUnitAttemptId queryUnitAttemptId, TaskAttemptState state, float progress,
+ public TaskHistory(TaskAttemptId taskAttemptId, TaskAttemptState state, float progress,
long startTime, long finishTime, CatalogProtos.TableStatsProto inputStats) {
init();
- this.queryUnitAttemptId = queryUnitAttemptId;
+ this.taskAttemptId = taskAttemptId;
this.state = state;
this.progress = progress;
this.startTime = startTime;
@@ -63,7 +63,7 @@ public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
}
public TaskHistory(TaskHistoryProto proto) {
- this.queryUnitAttemptId = new QueryUnitAttemptId(proto.getQueryUnitAttemptId());
+ this.taskAttemptId = new TaskAttemptId(proto.getTaskAttemptId());
this.state = proto.getState();
this.progress = proto.getProgress();
this.startTime = proto.getStartTime();
@@ -99,7 +99,7 @@ public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
@Override
public int hashCode() {
- return Objects.hashCode(queryUnitAttemptId, state);
+ return Objects.hashCode(taskAttemptId, state);
}
@Override
@@ -114,7 +114,7 @@ public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
@Override
public TaskHistoryProto getProto() {
TaskHistoryProto.Builder builder = TaskHistoryProto.newBuilder();
- builder.setQueryUnitAttemptId(queryUnitAttemptId.getProto());
+ builder.setTaskAttemptId(taskAttemptId.getProto());
builder.setState(state);
builder.setProgress(progress);
builder.setStartTime(startTime);
@@ -158,8 +158,8 @@ public class TaskHistory implements ProtoObject<TaskHistoryProto>, History {
fetcherHistories.add(fetcherHistory);
}
- public QueryUnitAttemptId getQueryUnitAttemptId() {
- return queryUnitAttemptId;
+ public TaskAttemptId getTaskAttemptId() {
+ return taskAttemptId;
}
public TaskAttemptState getState() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
index 4e9860b..f0da0cd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
@@ -25,10 +25,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.engine.query.QueryUnitRequestImpl;
+import org.apache.tajo.engine.query.TaskRequestImpl;
import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.container.impl.pb.TajoContainerIdPBImpl;
@@ -42,7 +42,7 @@ import java.util.concurrent.*;
import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
/**
- * The driver class for Tajo QueryUnit processing.
+ * The driver class for Tajo Task processing.
*/
public class TaskRunner extends AbstractService {
/** class logger */
@@ -165,7 +165,7 @@ public class TaskRunner extends AbstractService {
}
static void fatalError(QueryMasterProtocolService.Interface qmClientService,
- QueryUnitAttemptId taskAttemptId, String message) {
+ TaskAttemptId taskAttemptId, String message) {
if (message == null) {
message = "No error message";
}
@@ -185,8 +185,8 @@ public class TaskRunner extends AbstractService {
@Override
public void run() {
int receivedNum = 0;
- CallFuture<QueryUnitRequestProto> callFuture = null;
- QueryUnitRequestProto taskRequest = null;
+ CallFuture<TaskRequestProto> callFuture = null;
+ TaskRequestProto taskRequest = null;
while(!stopped) {
QueryMasterProtocolService.Interface qmClientService;
@@ -207,7 +207,7 @@ public class TaskRunner extends AbstractService {
try {
if (callFuture == null) {
- callFuture = new CallFuture<QueryUnitRequestProto>();
+ callFuture = new CallFuture<TaskRequestProto>();
LOG.info("Request GetTask: " + getId());
GetTaskRequestProto request = GetTaskRequestProto.newBuilder()
.setExecutionBlockId(getExecutionBlockId().getProto())
@@ -254,7 +254,7 @@ public class TaskRunner extends AbstractService {
getContext().getWorkerContext().getWorkerSystemMetrics().counter("query", "task").inc();
LOG.info("Accumulated Received Task: " + (++receivedNum));
- QueryUnitAttemptId taskAttemptId = new QueryUnitAttemptId(taskRequest.getId());
+ TaskAttemptId taskAttemptId = new TaskAttemptId(taskRequest.getId());
if (getContext().getTasks().containsKey(taskAttemptId)) {
LOG.error("Duplicate Task Attempt: " + taskAttemptId);
fatalError(qmClientService, taskAttemptId, "Duplicate Task Attempt: " + taskAttemptId);
@@ -265,7 +265,7 @@ public class TaskRunner extends AbstractService {
Task task;
try {
task = new Task(getId(), getTaskBaseDir(), taskAttemptId, executionBlockContext,
- new QueryUnitRequestImpl(taskRequest));
+ new TaskRequestImpl(taskRequest));
getContext().getTasks().put(taskAttemptId, task);
task.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
index 364348f..5c97ba8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
@@ -22,7 +22,7 @@ import com.google.common.base.Objects;
import com.google.common.collect.Maps;
import org.apache.hadoop.service.Service;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.container.TajoConverterUtils;
@@ -43,7 +43,7 @@ public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
private long startTime;
private long finishTime;
private ExecutionBlockId executionBlockId;
- private Map<QueryUnitAttemptId, TaskHistory> taskHistoryMap = null;
+ private Map<TaskAttemptId, TaskHistory> taskHistoryMap = null;
public TaskRunnerHistory(TajoContainerId containerId, ExecutionBlockId executionBlockId) {
init();
@@ -60,7 +60,7 @@ public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
this.taskHistoryMap = Maps.newTreeMap();
for (TaskHistoryProto taskHistoryProto : proto.getTaskHistoriesList()) {
TaskHistory taskHistory = new TaskHistory(taskHistoryProto);
- taskHistoryMap.put(taskHistory.getQueryUnitAttemptId(), taskHistory);
+ taskHistoryMap.put(taskHistory.getTaskAttemptId(), taskHistory);
}
}
@@ -137,15 +137,15 @@ public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
this.containerId = containerId;
}
- public TaskHistory getTaskHistory(QueryUnitAttemptId queryUnitAttemptId) {
- return taskHistoryMap.get(queryUnitAttemptId);
+ public TaskHistory getTaskHistory(TaskAttemptId taskAttemptId) {
+ return taskHistoryMap.get(taskAttemptId);
}
- public Map<QueryUnitAttemptId, TaskHistory> getTaskHistoryMap() {
+ public Map<TaskAttemptId, TaskHistory> getTaskHistoryMap() {
return Collections.unmodifiableMap(taskHistoryMap);
}
- public void addTaskHistory(QueryUnitAttemptId queryUnitAttemptId, TaskHistory taskHistory) {
- taskHistoryMap.put(queryUnitAttemptId, taskHistory);
+ public void addTaskHistory(TaskAttemptId taskAttemptId, TaskHistory taskHistory) {
+ taskHistoryMap.put(taskAttemptId, taskHistory);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
index a06e6e2..f837b11 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.utils.TupleCache;
import org.apache.tajo.worker.event.TaskRunnerEvent;
@@ -137,15 +137,15 @@ public class TaskRunnerManager extends CompositeService implements EventHandler<
return taskRunnerMap.get(taskRunnerId);
}
- public Task getTaskByQueryUnitAttemptId(QueryUnitAttemptId queryUnitAttemptId) {
- ExecutionBlockContext context = executionBlockContextMap.get(queryUnitAttemptId.getQueryUnitId().getExecutionBlockId());
+ public Task getTaskByTaskAttemptId(TaskAttemptId taskAttemptId) {
+ ExecutionBlockContext context = executionBlockContextMap.get(taskAttemptId.getTaskId().getExecutionBlockId());
if (context != null) {
- return context.getTask(queryUnitAttemptId);
+ return context.getTask(taskAttemptId);
}
return null;
}
- public TaskHistory getTaskHistoryByQueryUnitAttemptId(QueryUnitAttemptId quAttemptId) {
+ public TaskHistory getTaskHistoryByTaskAttemptId(TaskAttemptId quAttemptId) {
synchronized (taskRunnerHistoryMap) {
for (TaskRunnerHistory history : taskRunnerHistoryMap.values()) {
TaskHistory taskHistory = history.getTaskHistory(quAttemptId);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/worker/dataserver/retriever/AdvancedDataRetriever.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/dataserver/retriever/AdvancedDataRetriever.java b/tajo-core/src/main/java/org/apache/tajo/worker/dataserver/retriever/AdvancedDataRetriever.java
index 2ef0c4c..9c15d0c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/dataserver/retriever/AdvancedDataRetriever.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/dataserver/retriever/AdvancedDataRetriever.java
@@ -23,8 +23,8 @@ import com.google.common.collect.Maps;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.util.TajoIdUtils;
import org.apache.tajo.worker.dataserver.FileAccessForbiddenException;
import org.jboss.netty.channel.ChannelHandlerContext;
@@ -46,7 +46,7 @@ public class AdvancedDataRetriever implements DataRetriever {
public AdvancedDataRetriever() {
}
- public void register(QueryUnitAttemptId id, RetrieverHandler handler) {
+ public void register(TaskAttemptId id, RetrieverHandler handler) {
synchronized (handlerMap) {
if (!handlerMap.containsKey(id.toString())) {
handlerMap.put(id.toString(), handler);
@@ -54,7 +54,7 @@ public class AdvancedDataRetriever implements DataRetriever {
}
}
- public void unregister(QueryUnitAttemptId id) {
+ public void unregister(TaskAttemptId id) {
synchronized (handlerMap) {
if (handlerMap.containsKey(id.toString())) {
handlerMap.remove(id.toString());
@@ -82,8 +82,8 @@ public class AdvancedDataRetriever implements DataRetriever {
for (String qid : qids) {
String[] ids = qid.split("_");
ExecutionBlockId suid = TajoIdUtils.createExecutionBlockId(params.get("sid").get(0));
- QueryUnitId quid = new QueryUnitId(suid, Integer.parseInt(ids[0]));
- QueryUnitAttemptId attemptId = new QueryUnitAttemptId(quid,
+ TaskId quid = new TaskId(suid, Integer.parseInt(ids[0]));
+ TaskAttemptId attemptId = new TaskAttemptId(quid,
Integer.parseInt(ids[1]));
RetrieverHandler handler = handlerMap.get(attemptId.toString());
FileChunk chunk = handler.get(params);
@@ -115,7 +115,7 @@ public class AdvancedDataRetriever implements DataRetriever {
private List<String> splitMaps(List<String> qids) {
if (null == qids) {
- LOG.error("QueryUnitId is EMPTY");
+ LOG.error("QueryId is EMPTY");
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/proto/QueryMasterProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/QueryMasterProtocol.proto b/tajo-core/src/main/proto/QueryMasterProtocol.proto
index 494d296..ae20309 100644
--- a/tajo-core/src/main/proto/QueryMasterProtocol.proto
+++ b/tajo-core/src/main/proto/QueryMasterProtocol.proto
@@ -33,7 +33,7 @@ package hadoop.yarn;
service QueryMasterProtocolService {
//from Worker
- rpc getTask(GetTaskRequestProto) returns (QueryUnitRequestProto);
+ rpc getTask(GetTaskRequestProto) returns (TaskRequestProto);
rpc statusUpdate (TaskStatusProto) returns (BoolProto);
rpc ping (ExecutionBlockIdProto) returns (BoolProto);
rpc fatalError(TaskFatalErrorReport) returns (BoolProto);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index 989b0e3..5acbcd9 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -41,7 +41,7 @@ message SessionProto {
}
message TaskStatusProto {
- required QueryUnitAttemptIdProto id = 1;
+ required TaskAttemptIdProto id = 1;
required string workerName = 2;
required float progress = 3;
required TaskAttemptState state = 4;
@@ -52,7 +52,7 @@ message TaskStatusProto {
}
message TaskCompletionReport {
- required QueryUnitAttemptIdProto id = 1;
+ required TaskAttemptIdProto id = 1;
optional StatSetProto stats = 2;
optional TableStatsProto inputStats = 3;
optional TableStatsProto resultStats = 4;
@@ -60,13 +60,13 @@ message TaskCompletionReport {
}
message TaskFatalErrorReport {
- required QueryUnitAttemptIdProto id = 1;
+ required TaskAttemptIdProto id = 1;
optional string errorMessage = 2;
optional string errorTrace = 3;
}
-message QueryUnitRequestProto {
- required QueryUnitAttemptIdProto id = 1;
+message TaskRequestProto {
+ required TaskAttemptIdProto id = 1;
repeated FragmentProto fragments = 2;
required string outputTable = 3;
required bool clusteredOutput = 4;
@@ -126,7 +126,7 @@ message ExecutionBlockReport {
repeated IntermediateEntryProto intermediateEntries = 5;
}
-message QueryUnitResponseProto {
+message TaskResponseProto {
required string id = 1;
required QueryState status = 2;
}
@@ -135,7 +135,7 @@ message StatusReportProto {
required int64 timestamp = 1;
required string serverName = 2;
repeated TaskStatusProto status = 3;
- repeated QueryUnitAttemptIdProto pings = 4;
+ repeated TaskAttemptIdProto pings = 4;
}
message CommandRequestProto {
@@ -146,7 +146,7 @@ message CommandResponseProto {
}
message Command {
- required QueryUnitAttemptIdProto id = 1;
+ required TaskAttemptIdProto id = 1;
required CommandType type = 2;
}
@@ -208,12 +208,12 @@ message ExecutionBlockListProto {
}
service TajoWorkerProtocolService {
- rpc ping (QueryUnitAttemptIdProto) returns (BoolProto);
+ rpc ping (TaskAttemptIdProto) returns (BoolProto);
// from QueryMaster(Worker)
rpc startExecutionBlock(RunExecutionBlockRequestProto) returns (BoolProto);
rpc stopExecutionBlock(ExecutionBlockIdProto) returns (BoolProto);
- rpc killTaskAttempt(QueryUnitAttemptIdProto) returns (BoolProto);
+ rpc killTaskAttempt(TaskAttemptIdProto) returns (BoolProto);
rpc cleanup(QueryIdProto) returns (BoolProto);
rpc cleanupExecutionBlocks(ExecutionBlockListProto) returns (BoolProto);
}
@@ -336,7 +336,7 @@ message FetcherHistoryProto {
}
message TaskHistoryProto {
- required QueryUnitAttemptIdProto queryUnitAttemptId = 1;
+ required TaskAttemptIdProto taskAttemptId = 1;
required TaskAttemptState state = 2;
required float progress = 3;
required int64 startTime = 4;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
index 7a23157..ed97eff 100644
--- a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp
@@ -31,7 +31,7 @@
<%@ page import="org.apache.tajo.util.history.SubQueryHistory" %>
<%@ page import="org.apache.tajo.master.rm.Worker" %>
<%@ page import="java.util.*" %>
-<%@ page import="org.apache.tajo.util.history.QueryUnitHistory" %>
+<%@ page import="org.apache.tajo.util.history.TaskHistory" %>
<%
TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
@@ -100,15 +100,15 @@
totalWriteRows = subQuery.getTotalWriteRows();
}
- List<QueryUnitHistory> allQueryUnits = reader.getQueryUnitHistory(queryId, ebId);
- int numTasks = allQueryUnits.size();
+ List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId);
+ int numTasks = allTasks.size();
int numShuffles = 0;
float totalProgress = 0.0f;
- if (allQueryUnits != null) {
- for(QueryUnitHistory eachQueryUnit: allQueryUnits) {
- totalProgress += eachQueryUnit.getProgress();
- numShuffles = eachQueryUnit.getNumShuffles();
+ if (allTasks != null) {
+ for(TaskHistory eachTask: allTasks) {
+ totalProgress += eachTask.getProgress();
+ numShuffles = eachTask.getNumShuffles();
}
}
@@ -187,53 +187,53 @@
<input type="hidden" name="startTime" value="<%=startTime%>"/>
</form>
<%
- List<QueryUnitHistory> filteredQueryUnit = new ArrayList<QueryUnitHistory>();
- for(QueryUnitHistory eachQueryUnit: allQueryUnits) {
+ List<TaskHistory> filteredTasks = new ArrayList<TaskHistory>();
+ for(TaskHistory eachTask: allTasks) {
if (!"ALL".equals(status)) {
- if (!status.equals(eachQueryUnit.getState().toString())) {
+ if (!status.equals(eachTask.getState().toString())) {
continue;
}
}
- filteredQueryUnit.add(eachQueryUnit);
+ filteredTasks.add(eachTask);
}
- JSPUtil.sortQueryUnitHistory(filteredQueryUnit, sort, sortOrder);
- List<QueryUnitHistory> queryUnits = JSPUtil.getPageNavigationList(filteredQueryUnit, currentPage, pageSize);
+ JSPUtil.sortTaskHistory(filteredTasks, sort, sortOrder);
+ List<TaskHistory> tasks = JSPUtil.getPageNavigationList(filteredTasks, currentPage, pageSize);
- int numOfQueryUnits = filteredQueryUnit.size();
- int totalPage = numOfQueryUnits % pageSize == 0 ?
- numOfQueryUnits / pageSize : numOfQueryUnits / pageSize + 1;
+ int numOfTasks = filteredTasks.size();
+ int totalPage = numOfTasks % pageSize == 0 ?
+ numOfTasks / pageSize : numOfTasks / pageSize + 1;
%>
- <div align="right"># Tasks: <%=numOfQueryUnits%> / # Pages: <%=totalPage%></div>
+ <div align="right"># Tasks: <%=numOfTasks%> / # Pages: <%=totalPage%></div>
<table border="1" width="100%" class="border_table">
<tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th><a href='<%=url%>host'>Host</a></th></tr>
<%
int rowNo = (currentPage - 1) * pageSize + 1;
- for (QueryUnitHistory eachQueryUnit: queryUnits) {
- String queryUnitDetailUrl = "";
- if (eachQueryUnit.getId() != null) {
- queryUnitDetailUrl = "queryunit.jsp?queryId=" + queryId + "&ebid=" + ebId + "&startTime=" + startTime +
- "&queryUnitAttemptId=" + eachQueryUnit.getId() + "&sort=" + sort + "&sortOrder=" + sortOrder;
+ for (TaskHistory eachTask: tasks) {
+ String taskDetailUrl = "";
+ if (eachTask.getId() != null) {
+ taskDetailUrl = "task.jsp?queryId=" + queryId + "&ebid=" + ebId + "&startTime=" + startTime +
+ "&taskAttemptId=" + eachTask.getId() + "&sort=" + sort + "&sortOrder=" + sortOrder;
}
- String queryUnitHost = eachQueryUnit.getHostAndPort() == null ? "-" : eachQueryUnit.getHostAndPort();
- if (eachQueryUnit.getHostAndPort() != null) {
- Worker worker = workerMap.get(eachQueryUnit.getHostAndPort());
+ String taskHost = eachTask.getHostAndPort() == null ? "-" : eachTask.getHostAndPort();
+ if (eachTask.getHostAndPort() != null) {
+ Worker worker = workerMap.get(eachTask.getHostAndPort());
if (worker != null) {
- String[] hostTokens = eachQueryUnit.getHostAndPort().split(":");
- queryUnitHost = "<a href='http://" + hostTokens[0] + ":" + worker.getConnectionInfo().getHttpInfoPort() +
- "/taskhistory.jsp?queryUnitAttemptId=" + eachQueryUnit.getId() + "&startTime=" + eachQueryUnit.getLaunchTime() +
- "'>" + eachQueryUnit.getHostAndPort() + "</a>";
+ String[] hostTokens = eachTask.getHostAndPort().split(":");
+ taskHost = "<a href='http://" + hostTokens[0] + ":" + worker.getConnectionInfo().getHttpInfoPort() +
+ "/taskhistory.jsp?taskAttemptId=" + eachTask.getId() + "&startTime=" + eachTask.getLaunchTime() +
+ "'>" + eachTask.getHostAndPort() + "</a>";
}
}
%>
<tr>
<td><%=rowNo%></td>
- <td><a href="<%=queryUnitDetailUrl%>"><%=eachQueryUnit.getId()%></a></td>
- <td><%=eachQueryUnit.getState()%></td>
- <td><%=JSPUtil.percentFormat(eachQueryUnit.getProgress())%>%</td>
- <td><%=eachQueryUnit.getLaunchTime() == 0 ? "-" : df.format(eachQueryUnit.getLaunchTime())%></td>
- <td align='right'><%=eachQueryUnit.getLaunchTime() == 0 ? "-" : eachQueryUnit.getRunningTime() + " ms"%></td>
- <td><%=queryUnitHost%></td>
+ <td><a href="<%=taskDetailUrl%>"><%=eachTask.getId()%></a></td>
+ <td><%=eachTask.getState()%></td>
+ <td><%=JSPUtil.percentFormat(eachTask.getProgress())%>%</td>
+ <td><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
+ <td align='right'><%=eachTask.getLaunchTime() == 0 ? "-" : eachTask.getRunningTime() + " ms"%></td>
+ <td><%=taskHost%></td>
</tr>
<%
rowNo++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/admin/queryunit.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/queryunit.jsp b/tajo-core/src/main/resources/webapps/admin/queryunit.jsp
deleted file mode 100644
index 697469f..0000000
--- a/tajo-core/src/main/resources/webapps/admin/queryunit.jsp
+++ /dev/null
@@ -1,134 +0,0 @@
-<%
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-%>
-<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
-<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
-<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="java.text.SimpleDateFormat" %>
-<%@ page import="org.apache.tajo.master.TajoMaster" %>
-<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
-<%@ page import="org.apache.tajo.util.history.QueryUnitHistory" %>
-<%@ page import="java.util.List" %>
-
-<%
- TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
- HistoryReader reader = master.getContext().getHistoryReader();
-
- String queryId = request.getParameter("queryId");
- String ebId = request.getParameter("ebid");
-
- String status = request.getParameter("status");
- if(status == null || status.isEmpty() || "null".equals(status)) {
- status = "ALL";
- }
-
- String queryUnitAttemptId = request.getParameter("queryUnitAttemptId");
-
- List<QueryUnitHistory> allQueryUnits = reader.getQueryUnitHistory(queryId, ebId);
-
- QueryUnitHistory queryUnit = null;
- for(QueryUnitHistory eachQueryUnit: allQueryUnits) {
- if (eachQueryUnit.getId().equals(queryUnitAttemptId)) {
- queryUnit = eachQueryUnit;
- break;
- }
- }
-
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- String backUrl = request.getHeader("referer");
-%>
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
- <link rel="stylesheet" type="text/css" href="/static/style.css"/>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>Query Unit Detail</title>
-</head>
-<body>
-<%
- if (queryUnit == null) {
-%>
- <div>No QueryUnit history.</div>
- <div><a href="<%=backUrl%>">Back</a></div>
-<%
- return;
- }
-
- String fragmentInfo = "";
- String delim = "";
-
- for (String eachFragment : queryUnit.getFragments()) {
- fragmentInfo += delim + eachFragment;
- delim = "<br/>";
- }
-
- String fetchInfo = "";
- delim = "";
- String previousKey = null;
- for (String[] e : queryUnit.getFetchs()) {
- if (previousKey == null || !previousKey.equals(e[0])) {
- fetchInfo += delim + "<b>" + e[0] + "</b>";
- }
- delim = "<br/>";
- fetchInfo += delim + e[1];
-
- previousKey = e[0];
- }
-
- String dataLocationInfos = "";
- delim = "";
- for (String eachLocation: queryUnit.getDataLocations()) {
- dataLocationInfos += delim + eachLocation.toString();
- delim = "<br/>";
- }
-
- int numShuffles = queryUnit.getNumShuffles();
- String shuffleKey = "-";
- String shuffleFileName = "-";
- if(numShuffles > 0) {
- shuffleKey = queryUnit.getShuffleKey();
- shuffleFileName = queryUnit.getShuffleFileName();
- }
-%>
-
-
-<%@ include file="header.jsp"%>
-<div class='contents'>
- <h2>Tajo Master: <%=master.getMasterName()%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
- <hr/>
- <h3><a href='<%=backUrl%>'><%=ebId%></a></h3>
- <hr/>
- <table border="1" width="100%" class="border_table">
- <tr><td width="200" align="right">ID</td><td><%=queryUnit.getId()%></td></tr>
- <tr><td align="right">Progress</td><td><%=JSPUtil.percentFormat(queryUnit.getProgress())%>%</td></tr>
- <tr><td align="right">State</td><td><%=queryUnit.getState()%></td></tr>
- <tr><td align="right">Launch Time</td><td><%=queryUnit.getLaunchTime() == 0 ? "-" : df.format(queryUnit.getLaunchTime())%></td></tr>
- <tr><td align="right">Finish Time</td><td><%=queryUnit.getFinishTime() == 0 ? "-" : df.format(queryUnit.getFinishTime())%></td></tr>
- <tr><td align="right">Running Time</td><td><%=queryUnit.getLaunchTime() == 0 ? "-" : queryUnit.getRunningTime() + " ms"%></td></tr>
- <tr><td align="right">Host</td><td><%=queryUnit.getHostAndPort() == null ? "-" : queryUnit.getHostAndPort()%></td></tr>
- <tr><td align="right">Shuffles</td><td># Shuffle Outputs: <%=numShuffles%>, Shuffle Key: <%=shuffleKey%>, Shuffle file: <%=shuffleFileName%></td></tr>
- <tr><td align="right">Data Locations</td><td><%=dataLocationInfos%></td></tr>
- <tr><td align="right">Fragment</td><td><%=fragmentInfo%></td></tr>
- <tr><td align="right">Fetches</td><td><%=fetchInfo%></td></tr>
- </table>
-</div>
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/admin/task.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/task.jsp b/tajo-core/src/main/resources/webapps/admin/task.jsp
new file mode 100644
index 0000000..1530572
--- /dev/null
+++ b/tajo-core/src/main/resources/webapps/admin/task.jsp
@@ -0,0 +1,134 @@
+<%
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
+
+<%@ page import="org.apache.tajo.util.JSPUtil" %>
+<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
+<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
+<%@ page import="java.text.SimpleDateFormat" %>
+<%@ page import="org.apache.tajo.master.TajoMaster" %>
+<%@ page import="org.apache.tajo.util.history.HistoryReader" %>
+<%@ page import="org.apache.tajo.util.history.TaskHistory" %>
+<%@ page import="java.util.List" %>
+
+<%
+ TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
+ HistoryReader reader = master.getContext().getHistoryReader();
+
+ String queryId = request.getParameter("queryId");
+ String ebId = request.getParameter("ebid");
+
+ String status = request.getParameter("status");
+ if(status == null || status.isEmpty() || "null".equals(status)) {
+ status = "ALL";
+ }
+
+ String taskAttemptId = request.getParameter("taskAttemptId");
+
+ List<TaskHistory> allTasks = reader.getTaskHistory(queryId, ebId);
+
+ TaskHistory task = null;
+ for(TaskHistory eachTask: allTasks) {
+ if (eachTask.getId().equals(taskAttemptId)) {
+ task = eachTask;
+ break;
+ }
+ }
+
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String backUrl = request.getHeader("referer");
+%>
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+ <link rel="stylesheet" type="text/css" href="/static/style.css"/>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+ <title>Query Unit Detail</title>
+</head>
+<body>
+<%
+ if (task == null) {
+%>
+ <div>No Task history.</div>
+ <div><a href="<%=backUrl%>">Back</a></div>
+<%
+ return;
+ }
+
+ String fragmentInfo = "";
+ String delim = "";
+
+ for (String eachFragment : task.getFragments()) {
+ fragmentInfo += delim + eachFragment;
+ delim = "<br/>";
+ }
+
+ String fetchInfo = "";
+ delim = "";
+ String previousKey = null;
+ for (String[] e : task.getFetchs()) {
+ if (previousKey == null || !previousKey.equals(e[0])) {
+ fetchInfo += delim + "<b>" + e[0] + "</b>";
+ }
+ delim = "<br/>";
+ fetchInfo += delim + e[1];
+
+ previousKey = e[0];
+ }
+
+ String dataLocationInfos = "";
+ delim = "";
+ for (String eachLocation: task.getDataLocations()) {
+ dataLocationInfos += delim + eachLocation.toString();
+ delim = "<br/>";
+ }
+
+ int numShuffles = task.getNumShuffles();
+ String shuffleKey = "-";
+ String shuffleFileName = "-";
+ if(numShuffles > 0) {
+ shuffleKey = task.getShuffleKey();
+ shuffleFileName = task.getShuffleFileName();
+ }
+%>
+
+
+<%@ include file="header.jsp"%>
+<div class='contents'>
+ <h2>Tajo Master: <%=master.getMasterName()%> <%=JSPUtil.getMasterActiveLabel(master.getContext())%></h2>
+ <hr/>
+ <h3><a href='<%=backUrl%>'><%=ebId%></a></h3>
+ <hr/>
+ <table border="1" width="100%" class="border_table">
+ <tr><td width="200" align="right">ID</td><td><%=task.getId()%></td></tr>
+ <tr><td align="right">Progress</td><td><%=JSPUtil.percentFormat(task.getProgress())%>%</td></tr>
+ <tr><td align="right">State</td><td><%=task.getState()%></td></tr>
+ <tr><td align="right">Launch Time</td><td><%=task.getLaunchTime() == 0 ? "-" : df.format(task.getLaunchTime())%></td></tr>
+ <tr><td align="right">Finish Time</td><td><%=task.getFinishTime() == 0 ? "-" : df.format(task.getFinishTime())%></td></tr>
+ <tr><td align="right">Running Time</td><td><%=task.getLaunchTime() == 0 ? "-" : task.getRunningTime() + " ms"%></td></tr>
+ <tr><td align="right">Host</td><td><%=task.getHostAndPort() == null ? "-" : task.getHostAndPort()%></td></tr>
+ <tr><td align="right">Shuffles</td><td># Shuffle Outputs: <%=numShuffles%>, Shuffle Key: <%=shuffleKey%>, Shuffle file: <%=shuffleFileName%></td></tr>
+ <tr><td align="right">Data Locations</td><td><%=dataLocationInfos%></td></tr>
+ <tr><td align="right">Fragment</td><td><%=fragmentInfo%></td></tr>
+ <tr><td align="right">Fetches</td><td><%=fetchInfo%></td></tr>
+ </table>
+</div>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
index f39f57c..265937c 100644
--- a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
@@ -21,7 +21,7 @@
<%@ page import="org.apache.tajo.ExecutionBlockId" %>
<%@ page import="org.apache.tajo.QueryId" %>
-<%@ page import="org.apache.tajo.QueryUnitAttemptId" %>
+<%@ page import="org.apache.tajo.TaskAttemptId" %>
<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
<%@ page import="org.apache.tajo.plan.util.PlannerUtil" %>
<%@ page import="org.apache.tajo.ipc.TajoMasterProtocol" %>
@@ -97,30 +97,30 @@
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- QueryUnit[] allQueryUnits = subQuery.getQueryUnits();
+ Task[] allTasks = subQuery.getTasks();
long totalInputBytes = 0;
long totalReadBytes = 0;
long totalReadRows = 0;
long totalWriteBytes = 0;
long totalWriteRows = 0;
- int numTasks = allQueryUnits.length;
+ int numTasks = allTasks.length;
// int numSucceededTasks = 0;
// int localReadTasks = subQuery.;
int numShuffles = 0;
float totalProgress = 0.0f;
- for(QueryUnit eachQueryUnit: allQueryUnits) {
- totalProgress += eachQueryUnit.getLastAttempt() != null ? eachQueryUnit.getLastAttempt().getProgress(): 0.0f;
- numShuffles = eachQueryUnit.getShuffleOutpuNum();
- if (eachQueryUnit.getLastAttempt() != null) {
- TableStats inputStats = eachQueryUnit.getLastAttempt().getInputStats();
+ for(Task eachTask : allTasks) {
+ totalProgress += eachTask.getLastAttempt() != null ? eachTask.getLastAttempt().getProgress(): 0.0f;
+ numShuffles = eachTask.getShuffleOutpuNum();
+ if (eachTask.getLastAttempt() != null) {
+ TableStats inputStats = eachTask.getLastAttempt().getInputStats();
if (inputStats != null) {
totalInputBytes += inputStats.getNumBytes();
totalReadBytes += inputStats.getReadBytes();
totalReadRows += inputStats.getNumRows();
}
- TableStats outputStats = eachQueryUnit.getLastAttempt().getResultStats();
+ TableStats outputStats = eachTask.getLastAttempt().getResultStats();
if (outputStats != null) {
totalWriteBytes += outputStats.getNumBytes();
totalWriteRows += outputStats.getNumRows();
@@ -202,55 +202,55 @@
<input type="hidden" name="sortOrder" value="<%=sortOrder%>"/>
</form>
<%
- List<QueryUnit> filteredQueryUnit = new ArrayList<QueryUnit>();
- for(QueryUnit eachQueryUnit: allQueryUnits) {
+ List<Task> filteredTask = new ArrayList<Task>();
+ for(Task eachTask : allTasks) {
if (!"ALL".equals(status)) {
- if (!status.equals(eachQueryUnit.getLastAttemptStatus().toString())) {
+ if (!status.equals(eachTask.getLastAttemptStatus().toString())) {
continue;
}
}
- filteredQueryUnit.add(eachQueryUnit);
+ filteredTask.add(eachTask);
}
- JSPUtil.sortQueryUnit(filteredQueryUnit, sort, sortOrder);
- List<QueryUnit> queryUnits = JSPUtil.getPageNavigationList(filteredQueryUnit, currentPage, pageSize);
+ JSPUtil.sortTasks(filteredTask, sort, sortOrder);
+ List<Task> tasks = JSPUtil.getPageNavigationList(filteredTask, currentPage, pageSize);
- int numOfQueryUnits = filteredQueryUnit.size();
- int totalPage = numOfQueryUnits % pageSize == 0 ?
- numOfQueryUnits / pageSize : numOfQueryUnits / pageSize + 1;
+ int numOfTasks = filteredTask.size();
+ int totalPage = numOfTasks % pageSize == 0 ?
+ numOfTasks / pageSize : numOfTasks / pageSize + 1;
int rowNo = (currentPage - 1) * pageSize + 1;
%>
- <div align="right"># Tasks: <%=numOfQueryUnits%> / # Pages: <%=totalPage%></div>
+ <div align="right"># Tasks: <%=numOfTasks%> / # Pages: <%=totalPage%></div>
<table border="1" width="100%" class="border_table">
<tr><th>No</th><th><a href='<%=url%>id'>Id</a></th><th>Status</th><th>Progress</th><th><a href='<%=url%>startTime'>Started</a></th><th><a href='<%=url%>runTime'>Running Time</a></th><th><a href='<%=url%>host'>Host</a></th></tr>
<%
- for(QueryUnit eachQueryUnit: queryUnits) {
- int queryUnitSeq = eachQueryUnit.getId().getId();
- String queryUnitDetailUrl = "queryunit.jsp?queryId=" + paramQueryId + "&ebid=" + paramEbId +
+ for(Task eachTask : tasks) {
+ int taskSeq = eachTask.getId().getId();
+ String taskDetailUrl = "task.jsp?queryId=" + paramQueryId + "&ebid=" + paramEbId +
"&page=" + currentPage + "&pageSize=" + pageSize +
- "&queryUnitSeq=" + queryUnitSeq + "&sort=" + sort + "&sortOrder=" + sortOrder;
+ "&taskSeq=" + taskSeq + "&sort=" + sort + "&sortOrder=" + sortOrder;
- String queryUnitHost = eachQueryUnit.getSucceededHost() == null ? "-" : eachQueryUnit.getSucceededHost();
- if(eachQueryUnit.getSucceededHost() != null) {
+ String taskHost = eachTask.getSucceededHost() == null ? "-" : eachTask.getSucceededHost();
+ if(eachTask.getSucceededHost() != null) {
TajoMasterProtocol.WorkerResourceProto worker =
- workerMap.get(eachQueryUnit.getLastAttempt().getWorkerConnectionInfo().getId());
+ workerMap.get(eachTask.getLastAttempt().getWorkerConnectionInfo().getId());
if(worker != null) {
- QueryUnitAttempt lastAttempt = eachQueryUnit.getLastAttempt();
+ TaskAttempt lastAttempt = eachTask.getLastAttempt();
if(lastAttempt != null) {
- QueryUnitAttemptId lastAttemptId = lastAttempt.getId();
- queryUnitHost = "<a href='http://" + eachQueryUnit.getSucceededHost() + ":" + worker.getConnectionInfo().getHttpInfoPort() + "/taskdetail.jsp?queryUnitAttemptId=" + lastAttemptId + "'>" + eachQueryUnit.getSucceededHost() + "</a>";
+ TaskAttemptId lastAttemptId = lastAttempt.getId();
+ taskHost = "<a href='http://" + eachTask.getSucceededHost() + ":" + worker.getConnectionInfo().getHttpInfoPort() + "/taskdetail.jsp?taskAttemptId=" + lastAttemptId + "'>" + eachTask.getSucceededHost() + "</a>";
}
}
}
%>
<tr>
<td><%=rowNo%></td>
- <td><a href="<%=queryUnitDetailUrl%>"><%=eachQueryUnit.getId()%></a></td>
- <td><%=eachQueryUnit.getLastAttemptStatus()%></td>
- <td><%=JSPUtil.percentFormat(eachQueryUnit.getLastAttempt().getProgress())%>%</td>
- <td><%=eachQueryUnit.getLaunchTime() == 0 ? "-" : df.format(eachQueryUnit.getLaunchTime())%></td>
- <td align='right'><%=eachQueryUnit.getLaunchTime() == 0 ? "-" : eachQueryUnit.getRunningTime() + " ms"%></td>
- <td><%=queryUnitHost%></td>
+ <td><a href="<%=taskDetailUrl%>"><%=eachTask.getId()%></a></td>
+ <td><%=eachTask.getLastAttemptStatus()%></td>
+ <td><%=JSPUtil.percentFormat(eachTask.getLastAttempt().getProgress())%>%</td>
+ <td><%=eachTask.getLaunchTime() == 0 ? "-" : df.format(eachTask.getLaunchTime())%></td>
+ <td align='right'><%=eachTask.getLaunchTime() == 0 ? "-" : eachTask.getRunningTime() + " ms"%></td>
+ <td><%=taskHost%></td>
</tr>
<%
rowNo++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/queryunit.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/queryunit.jsp b/tajo-core/src/main/resources/webapps/worker/queryunit.jsp
deleted file mode 100644
index 49635d1..0000000
--- a/tajo-core/src/main/resources/webapps/worker/queryunit.jsp
+++ /dev/null
@@ -1,175 +0,0 @@
-<%
- /*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-%>
-<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-
-<%@ page import="org.apache.tajo.ExecutionBlockId" %>
-<%@ page import="org.apache.tajo.QueryId" %>
-<%@ page import="org.apache.tajo.QueryUnitId" %>
-<%@ page import="org.apache.tajo.catalog.proto.CatalogProtos" %>
-<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
-<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
-<%@ page import="org.apache.tajo.master.querymaster.Query" %>
-<%@ page import="org.apache.tajo.master.querymaster.QueryMasterTask" %>
-<%@ page import="org.apache.tajo.master.querymaster.QueryUnit" %>
-<%@ page import="org.apache.tajo.master.querymaster.SubQuery" %>
-<%@ page import="org.apache.tajo.storage.DataLocation" %>
-<%@ page import="org.apache.tajo.storage.fragment.FileFragment" %>
-<%@ page import="org.apache.tajo.storage.fragment.FragmentConvertor" %>
-<%@ page import="org.apache.tajo.util.JSPUtil" %>
-<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
-<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
-<%@ page import="org.apache.tajo.worker.FetchImpl" %>
-<%@ page import="org.apache.tajo.worker.TajoWorker" %>
-<%@ page import="java.net.URI" %>
-<%@ page import="java.text.SimpleDateFormat" %>
-<%@ page import="java.util.Map" %>
-<%@ page import="java.util.Set" %>
-<%@ page import="org.apache.tajo.storage.fragment.Fragment" %>
-
-<%
- String paramQueryId = request.getParameter("queryId");
- String paramEbId = request.getParameter("ebid");
- String status = request.getParameter("status");
- if(status == null || status.isEmpty() || "null".equals(status)) {
- status = "ALL";
- }
-
- QueryId queryId = TajoIdUtils.parseQueryId(paramQueryId);
- ExecutionBlockId ebid = TajoIdUtils.createExecutionBlockId(paramEbId);
-
- int queryUnitSeq = Integer.parseInt(request.getParameter("queryUnitSeq"));
- TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
- QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
- .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
-
- if(queryMasterTask == null) {
- out.write("<script type='text/javascript'>alert('no query'); history.back(0); </script>");
- return;
- }
-
- Query query = queryMasterTask.getQuery();
- SubQuery subQuery = query.getSubQuery(ebid);
-
- if(subQuery == null) {
- out.write("<script type='text/javascript'>alert('no sub-query'); history.back(0); </script>");
- return;
- }
-
- if(subQuery == null) {
-%>
-<script type="text/javascript">
- alert("No Execution Block for" + ebid);
- document.history.back();
-</script>
-<%
- return;
- }
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-
- QueryUnitId queryUnitId = new QueryUnitId(ebid, queryUnitSeq);
- QueryUnit queryUnit = subQuery.getQueryUnit(queryUnitId);
- if(queryUnit == null) {
-%>
-<script type="text/javascript">
- alert("No QueryUnit for" + queryUnitId);
- document.history.back();
-</script>
-<%
- return;
- }
-
- String sort = request.getParameter("sort");
- String sortOrder = request.getParameter("sortOrder");
-
- String backUrl = "querytasks.jsp?queryId=" + paramQueryId + "&ebid=" + paramEbId + "&sort=" + sort + "&sortOrder=" + sortOrder + "&status=" + status;
-
- String fragmentInfo = "";
- String delim = "";
- for (CatalogProtos.FragmentProto eachFragment : queryUnit.getAllFragments()) {
- Fragment fragment = FragmentConvertor.convert(tajoWorker.getConfig(), eachFragment);
- fragmentInfo += delim + fragment.toString();
- delim = "<br/>";
- }
-
- String fetchInfo = "";
- delim = "";
- for (Map.Entry<String, Set<FetchImpl>> e : queryUnit.getFetchMap().entrySet()) {
- fetchInfo += delim + "<b>" + e.getKey() + "</b>";
- delim = "<br/>";
- for (FetchImpl f : e.getValue()) {
- for (URI uri : f.getSimpleURIs()){
- fetchInfo += delim + uri;
- }
- }
- }
-
- String dataLocationInfos = "";
- delim = "";
- for(DataLocation eachLocation: queryUnit.getDataLocations()) {
- dataLocationInfos += delim + eachLocation.toString();
- delim = "<br/>";
- }
-
- int numShuffles = queryUnit.getShuffleOutpuNum();
- String shuffleKey = "-";
- String shuffleFileName = "-";
- if(numShuffles > 0) {
- TajoWorkerProtocol.ShuffleFileOutput shuffleFileOutputs = queryUnit.getShuffleFileOutputs().get(0);
- shuffleKey = "" + shuffleFileOutputs.getPartId();
- shuffleFileName = shuffleFileOutputs.getFileName();
- }
-
- //int numIntermediateData = queryUnit.getIntermediateData() == null ? 0 : queryUnit.getIntermediateData().size();
- TableStats inputStat = queryUnit.getLastAttempt().getInputStats();
- TableStats outputStat = queryUnit.getLastAttempt().getResultStats();
-%>
-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
-<html>
-<head>
- <link rel="stylesheet" type="text/css" href="/static/style.css"/>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
- <title>Query Unit Detail</title>
-</head>
-<body>
-<%@ include file="header.jsp"%>
-<div class='contents'>
- <h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
- <hr/>
- <h3><a href='<%=backUrl%>'><%=ebid.toString()%></a></h3>
- <hr/>
- <table border="1" width="100%" class="border_table">
- <tr><td width="200" align="right">ID</td><td><%=queryUnit.getId()%></td></tr>
- <tr><td align="right">Progress</td><td><%=JSPUtil.percentFormat(queryUnit.getLastAttempt().getProgress())%>%</td></tr>
- <tr><td align="right">State</td><td><%=queryUnit.getState()%></td></tr>
- <tr><td align="right">Launch Time</td><td><%=queryUnit.getLaunchTime() == 0 ? "-" : df.format(queryUnit.getLaunchTime())%></td></tr>
- <tr><td align="right">Finish Time</td><td><%=queryUnit.getFinishTime() == 0 ? "-" : df.format(queryUnit.getFinishTime())%></td></tr>
- <tr><td align="right">Running Time</td><td><%=queryUnit.getLaunchTime() == 0 ? "-" : queryUnit.getRunningTime() + " ms"%></td></tr>
- <tr><td align="right">Host</td><td><%=queryUnit.getSucceededHost() == null ? "-" : queryUnit.getSucceededHost()%></td></tr>
- <tr><td align="right">Shuffles</td><td># Shuffle Outputs: <%=numShuffles%>, Shuffle Key: <%=shuffleKey%>, Shuffle file: <%=shuffleFileName%></td></tr>
- <tr><td align="right">Data Locations</td><td><%=dataLocationInfos%></td></tr>
- <tr><td align="right">Fragment</td><td><%=fragmentInfo%></td></tr>
- <tr><td align="right">Input Statistics</td><td><%=JSPUtil.tableStatToString(inputStat)%></td></tr>
- <tr><td align="right">Output Statistics</td><td><%=JSPUtil.tableStatToString(outputStat)%></td></tr>
- <tr><td align="right">Fetches</td><td><%=fetchInfo%></td></tr>
- </table>
-</div>
-</body>
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/task.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/task.jsp b/tajo-core/src/main/resources/webapps/worker/task.jsp
new file mode 100644
index 0000000..5abbd8c
--- /dev/null
+++ b/tajo-core/src/main/resources/webapps/worker/task.jsp
@@ -0,0 +1,174 @@
+<%
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+%>
+<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
+
+<%@ page import="org.apache.tajo.ExecutionBlockId" %>
+<%@ page import="org.apache.tajo.QueryId" %>
+<%@ page import="org.apache.tajo.TaskId" %>
+<%@ page import="org.apache.tajo.catalog.proto.CatalogProtos" %>
+<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
+<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
+<%@ page import="org.apache.tajo.master.querymaster.Query" %>
+<%@ page import="org.apache.tajo.master.querymaster.QueryMasterTask" %>
+<%@ page import="org.apache.tajo.master.querymaster.Task" %>
+<%@ page import="org.apache.tajo.master.querymaster.SubQuery" %>
+<%@ page import="org.apache.tajo.storage.DataLocation" %>
+<%@ page import="org.apache.tajo.storage.fragment.FileFragment" %>
+<%@ page import="org.apache.tajo.storage.fragment.FragmentConvertor" %>
+<%@ page import="org.apache.tajo.util.JSPUtil" %>
+<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
+<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
+<%@ page import="org.apache.tajo.worker.FetchImpl" %>
+<%@ page import="org.apache.tajo.worker.TajoWorker" %>
+<%@ page import="java.net.URI" %>
+<%@ page import="java.text.SimpleDateFormat" %>
+<%@ page import="java.util.Map" %>
+<%@ page import="java.util.Set" %>
+<%@ page import="org.apache.tajo.storage.fragment.Fragment" %>
+
+<%
+ String paramQueryId = request.getParameter("queryId");
+ String paramEbId = request.getParameter("ebid");
+ String status = request.getParameter("status");
+ if(status == null || status.isEmpty() || "null".equals(status)) {
+ status = "ALL";
+ }
+
+ QueryId queryId = TajoIdUtils.parseQueryId(paramQueryId);
+ ExecutionBlockId ebid = TajoIdUtils.createExecutionBlockId(paramEbId);
+
+ int taskSeq = Integer.parseInt(request.getParameter("taskSeq"));
+ TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
+ QueryMasterTask queryMasterTask = tajoWorker.getWorkerContext()
+ .getQueryMasterManagerService().getQueryMaster().getQueryMasterTask(queryId, true);
+
+ if(queryMasterTask == null) {
+ out.write("<script type='text/javascript'>alert('no query'); history.back(0); </script>");
+ return;
+ }
+
+ Query query = queryMasterTask.getQuery();
+ SubQuery subQuery = query.getSubQuery(ebid);
+
+ if(subQuery == null) {
+ out.write("<script type='text/javascript'>alert('no sub-query'); history.back(0); </script>");
+ return;
+ }
+
+ if(subQuery == null) {
+%>
+<script type="text/javascript">
+ alert("No Execution Block for" + ebid);
+ document.history.back();
+</script>
+<%
+ return;
+ }
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+
+ TaskId taskId = new TaskId(ebid, taskSeq);
+ Task task = subQuery.getTask(taskId);
+ if(task == null) {
+%>
+<script type="text/javascript">
+ alert("No Task for" + taskId);
+ document.history.back();
+</script>
+<%
+ return;
+ }
+
+ String sort = request.getParameter("sort");
+ String sortOrder = request.getParameter("sortOrder");
+
+ String backUrl = "querytasks.jsp?queryId=" + paramQueryId + "&ebid=" + paramEbId + "&sort=" + sort + "&sortOrder=" + sortOrder + "&status=" + status;
+
+ String fragmentInfo = "";
+ String delim = "";
+ for (CatalogProtos.FragmentProto eachFragment : task.getAllFragments()) {
+ Fragment fragment = FragmentConvertor.convert(tajoWorker.getConfig(), eachFragment);
+ fragmentInfo += delim + fragment.toString();
+ delim = "<br/>";
+ }
+
+ String fetchInfo = "";
+ delim = "";
+ for (Map.Entry<String, Set<FetchImpl>> e : task.getFetchMap().entrySet()) {
+ fetchInfo += delim + "<b>" + e.getKey() + "</b>";
+ delim = "<br/>";
+ for (FetchImpl f : e.getValue()) {
+ for (URI uri : f.getSimpleURIs()){
+ fetchInfo += delim + uri;
+ }
+ }
+ }
+
+ String dataLocationInfos = "";
+ delim = "";
+ for(DataLocation eachLocation: task.getDataLocations()) {
+ dataLocationInfos += delim + eachLocation.toString();
+ delim = "<br/>";
+ }
+
+ int numShuffles = task.getShuffleOutpuNum();
+ String shuffleKey = "-";
+ String shuffleFileName = "-";
+ if(numShuffles > 0) {
+ TajoWorkerProtocol.ShuffleFileOutput shuffleFileOutputs = task.getShuffleFileOutputs().get(0);
+ shuffleKey = "" + shuffleFileOutputs.getPartId();
+ shuffleFileName = shuffleFileOutputs.getFileName();
+ }
+
+ TableStats inputStat = task.getLastAttempt().getInputStats();
+ TableStats outputStat = task.getLastAttempt().getResultStats();
+%>
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+ <link rel="stylesheet" type="text/css" href="/static/style.css"/>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+ <title>Query Unit Detail</title>
+</head>
+<body>
+<%@ include file="header.jsp"%>
+<div class='contents'>
+ <h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
+ <hr/>
+ <h3><a href='<%=backUrl%>'><%=ebid.toString()%></a></h3>
+ <hr/>
+ <table border="1" width="100%" class="border_table">
+ <tr><td width="200" align="right">ID</td><td><%=task.getId()%></td></tr>
+ <tr><td align="right">Progress</td><td><%=JSPUtil.percentFormat(task.getLastAttempt().getProgress())%>%</td></tr>
+ <tr><td align="right">State</td><td><%=task.getState()%></td></tr>
+ <tr><td align="right">Launch Time</td><td><%=task.getLaunchTime() == 0 ? "-" : df.format(task.getLaunchTime())%></td></tr>
+ <tr><td align="right">Finish Time</td><td><%=task.getFinishTime() == 0 ? "-" : df.format(task.getFinishTime())%></td></tr>
+ <tr><td align="right">Running Time</td><td><%=task.getLaunchTime() == 0 ? "-" : task.getRunningTime() + " ms"%></td></tr>
+ <tr><td align="right">Host</td><td><%=task.getSucceededHost() == null ? "-" : task.getSucceededHost()%></td></tr>
+ <tr><td align="right">Shuffles</td><td># Shuffle Outputs: <%=numShuffles%>, Shuffle Key: <%=shuffleKey%>, Shuffle file: <%=shuffleFileName%></td></tr>
+ <tr><td align="right">Data Locations</td><td><%=dataLocationInfos%></td></tr>
+ <tr><td align="right">Fragment</td><td><%=fragmentInfo%></td></tr>
+ <tr><td align="right">Input Statistics</td><td><%=JSPUtil.tableStatToString(inputStat)%></td></tr>
+ <tr><td align="right">Output Statistics</td><td><%=JSPUtil.tableStatToString(outputStat)%></td></tr>
+ <tr><td align="right">Fetches</td><td><%=fetchInfo%></td></tr>
+ </table>
+</div>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp b/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
index d84664f..5c3ce7b 100644
--- a/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/taskdetail.jsp
@@ -20,7 +20,7 @@
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
<%@ page import="org.apache.commons.lang.StringUtils" %>
-<%@ page import="org.apache.tajo.QueryUnitAttemptId" %>
+<%@ page import="org.apache.tajo.TaskAttemptId" %>
<%@ page import="org.apache.tajo.ipc.TajoWorkerProtocol" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.util.TajoIdUtils" %>
@@ -33,27 +33,27 @@
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
String containerId = request.getParameter("containerId");
- String quAttemptId = request.getParameter("queryUnitAttemptId");
- QueryUnitAttemptId queryUnitAttemptId = TajoIdUtils.parseQueryUnitAttemptId(quAttemptId);
+ String quAttemptId = request.getParameter("taskAttemptId");
+ TaskAttemptId taskAttemptId = TajoIdUtils.parseTaskAttemptId(quAttemptId);
Task task = null;
TaskHistory taskHistory = null;
if(containerId == null || containerId.isEmpty() || "null".equals(containerId)) {
- task = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskByQueryUnitAttemptId(queryUnitAttemptId);
+ task = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskByTaskAttemptId(taskAttemptId);
if (task != null) {
taskHistory = task.createTaskHistory();
} else {
- taskHistory = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskHistoryByQueryUnitAttemptId(queryUnitAttemptId);
+ taskHistory = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskHistoryByTaskAttemptId(taskAttemptId);
}
} else {
TaskRunner runner = tajoWorker.getWorkerContext().getTaskRunnerManager().getTaskRunner(containerId);
if(runner != null) {
- task = runner.getContext().getTask(queryUnitAttemptId);
+ task = runner.getContext().getTask(taskAttemptId);
if (task != null) {
taskHistory = task.createTaskHistory();
} else {
TaskRunnerHistory history = tajoWorker.getWorkerContext().getTaskRunnerManager().getExcutionBlockHistoryByTaskRunnerId(containerId);
if(history != null) {
- taskHistory = history.getTaskHistory(queryUnitAttemptId);
+ taskHistory = history.getTaskHistory(taskAttemptId);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp b/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
index b777d5f..b7774e8 100644
--- a/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/taskhistory.jsp
@@ -31,7 +31,7 @@
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
HistoryReader reader = new HistoryReader(tajoWorker.getWorkerContext().getWorkerName(), tajoWorker.getWorkerContext().getConf());
- TaskHistory taskHistory = reader.getTaskHistory(request.getParameter("queryUnitAttemptId"),
+ TaskHistory taskHistory = reader.getTaskHistory(request.getParameter("taskAttemptId"),
Long.parseLong(request.getParameter("startTime")));
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -62,9 +62,9 @@
<div class='contents'>
<h2>Tajo Worker: <a href='index.jsp'><%=tajoWorker.getWorkerContext().getWorkerName()%></a></h2>
<hr/>
- <h3>Task Detail: <%=request.getParameter("queryUnitAttemptId")%></h3>
+ <h3>Task Detail: <%=request.getParameter("taskAttemptId")%></h3>
<table border="1" width="100%" class="border_table">
- <tr><td width="200" align="right">ID</td><td><%=request.getParameter("queryUnitAttemptId")%></td></tr>
+ <tr><td width="200" align="right">ID</td><td><%=request.getParameter("taskAttemptId")%></td></tr>
<tr><td align="right">State</td><td><%=taskHistory.getState()%></td></tr>
<tr><td align="right">Start Time</td><td><%=taskHistory.getStartTime() == 0 ? "-" : df.format(taskHistory.getStartTime())%></td></tr>
<tr><td align="right">Finish Time</td><td><%=taskHistory.getFinishTime() == 0 ? "-" : df.format(taskHistory.getFinishTime())%></td></tr>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/resources/webapps/worker/tasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/tasks.jsp b/tajo-core/src/main/resources/webapps/worker/tasks.jsp
index ae05047..ab873cd 100644
--- a/tajo-core/src/main/resources/webapps/worker/tasks.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/tasks.jsp
@@ -19,7 +19,7 @@
%>
<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%>
-<%@ page import="org.apache.tajo.QueryUnitAttemptId" %>
+<%@ page import="org.apache.tajo.TaskAttemptId" %>
<%@ page import="org.apache.tajo.util.JSPUtil" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="java.text.SimpleDateFormat" %>
@@ -67,13 +67,13 @@
if (taskRunner != null) {
ExecutionBlockContext context = taskRunner.getContext();
- for (Map.Entry<QueryUnitAttemptId, Task> entry : context.getTasks().entrySet()) {
- QueryUnitAttemptId queryUnitId = entry.getKey();
+ for (Map.Entry<TaskAttemptId, Task> entry : context.getTasks().entrySet()) {
+ TaskAttemptId taskAttemptId = entry.getKey();
TaskHistory eachTask = entry.getValue().createTaskHistory();
%>
<tr>
<td>
- <a href="taskdetail.jsp?containerId=<%=containerId%>&queryUnitAttemptId=<%=queryUnitId%>"><%=queryUnitId%></a></td>
+ <a href="taskdetail.jsp?containerId=<%=containerId%>&taskAttemptId=<%=taskAttemptId%>"><%=taskAttemptId%></a></td>
<td><%=df.format(eachTask.getStartTime())%></td>
<td><%=eachTask.getFinishTime() == 0 ? "-" : df.format(eachTask.getFinishTime())%></td>
<td><%=JSPUtil.getElapsedTime(eachTask.getStartTime(), eachTask.getFinishTime())%></td>
@@ -86,12 +86,12 @@
if (history != null) {
- for (Map.Entry<QueryUnitAttemptId, TaskHistory> entry : history.getTaskHistoryMap().entrySet()) {
- QueryUnitAttemptId queryUnitId = entry.getKey();
+ for (Map.Entry<TaskAttemptId, TaskHistory> entry : history.getTaskHistoryMap().entrySet()) {
+ TaskAttemptId taskAttemptId = entry.getKey();
TaskHistory eachTask = entry.getValue();
%>
<tr>
- <td><a href="taskdetail.jsp?containerId=<%=containerId%>&queryUnitAttemptId=<%=queryUnitId%>"><%=queryUnitId%></a></td>
+ <td><a href="taskdetail.jsp?containerId=<%=containerId%>&taskAttemptId=<%=taskAttemptId%>"><%=taskAttemptId%></a></td>
<td><%=df.format(eachTask.getStartTime())%></td>
<td><%=eachTask.getFinishTime() == 0 ? "-" : df.format(eachTask.getFinishTime())%></td>
<td><%=JSPUtil.getElapsedTime(eachTask.getStartTime(), eachTask.getFinishTime())%></td>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index f6a5fed..8bee6fb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -66,12 +66,12 @@ public class LocalTajoTestingUtility {
private static int taskAttemptId;
- public static QueryUnitAttemptId newQueryUnitAttemptId() {
- return QueryIdFactory.newQueryUnitAttemptId(
- QueryIdFactory.newQueryUnitId(new MasterPlan(newQueryId(), null, null).newExecutionBlockId()), taskAttemptId++);
+ public static TaskAttemptId newTaskAttemptId() {
+ return QueryIdFactory.newTaskAttemptId(
+ QueryIdFactory.newTaskId(new MasterPlan(newQueryId(), null, null).newExecutionBlockId()), taskAttemptId++);
}
- public static QueryUnitAttemptId newQueryUnitAttemptId(MasterPlan plan) {
- return QueryIdFactory.newQueryUnitAttemptId(QueryIdFactory.newQueryUnitId(plan.newExecutionBlockId()), 0);
+ public static TaskAttemptId newTaskAttemptId(MasterPlan plan) {
+ return QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(plan.newExecutionBlockId()), 0);
}
public static Session createDummySession() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/TestQueryIdFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TestQueryIdFactory.java b/tajo-core/src/test/java/org/apache/tajo/TestQueryIdFactory.java
index 912400b..836dd2f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TestQueryIdFactory.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TestQueryIdFactory.java
@@ -47,12 +47,12 @@ public class TestQueryIdFactory {
}
@Test
- public void testNewQueryUnitId() {
+ public void testNewTaskId() {
QueryId qid = LocalTajoTestingUtility.newQueryId();
MasterPlan plan = new MasterPlan(qid, null, null);
ExecutionBlockId subid = plan.newExecutionBlockId();
- QueryUnitId quid1 = QueryIdFactory.newQueryUnitId(subid);
- QueryUnitId quid2 = QueryIdFactory.newQueryUnitId(subid);
+ TaskId quid1 = QueryIdFactory.newTaskId(subid);
+ TaskId quid2 = QueryIdFactory.newTaskId(subid);
assertTrue(quid1.compareTo(quid2) < 0);
}
}
[09/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
TAJO-324: Rename the prefix 'QueryUnit' to Task.
Closes #306
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5c852b79
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5c852b79
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5c852b79
Branch: refs/heads/index_support
Commit: 5c852b7984ba96cd60d528dd2086e2745cc8e3f5
Parents: ff57c77
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Dec 18 12:55:25 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Dec 18 12:55:25 2014 +0900
----------------------------------------------------------------------
CHANGES | 5 +-
.../java/org/apache/tajo/QueryIdFactory.java | 12 +-
.../org/apache/tajo/QueryUnitAttemptId.java | 94 --
.../main/java/org/apache/tajo/QueryUnitId.java | 90 --
.../java/org/apache/tajo/TaskAttemptId.java | 94 ++
.../src/main/java/org/apache/tajo/TaskId.java | 90 ++
.../java/org/apache/tajo/util/TajoIdUtils.java | 8 +-
tajo-common/src/main/proto/TajoIdProtos.proto | 6 +-
.../physical/HashShuffleFileWriteExec.java | 3 +-
.../engine/planner/physical/PhysicalExec.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 2 +-
.../tajo/engine/query/QueryUnitRequest.java | 51 --
.../tajo/engine/query/QueryUnitRequestImpl.java | 328 -------
.../apache/tajo/engine/query/TaskRequest.java | 51 ++
.../tajo/engine/query/TaskRequestImpl.java | 328 +++++++
.../tajo/master/DefaultTaskScheduler.java | 162 ++--
.../apache/tajo/master/LazyTaskScheduler.java | 46 +-
.../master/NonForwardQueryResultScanner.java | 6 +-
.../apache/tajo/master/TajoContainerProxy.java | 4 +-
.../event/ContainerAllocatorEventType.java | 2 +-
.../tajo/master/event/LocalTaskEvent.java | 8 +-
.../event/QueryUnitAttemptScheduleEvent.java | 87 --
.../tajo/master/event/SubQueryEventType.java | 2 +-
.../tajo/master/event/SubQueryTaskEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 4 +-
.../tajo/master/event/TaskAttemptEvent.java | 8 +-
.../master/event/TaskAttemptScheduleEvent.java | 4 +-
.../event/TaskAttemptStatusUpdateEvent.java | 4 +-
.../event/TaskAttemptToSchedulerEvent.java | 87 ++
.../tajo/master/event/TaskCompletionEvent.java | 4 +-
.../org/apache/tajo/master/event/TaskEvent.java | 8 +-
.../tajo/master/event/TaskFatalErrorEvent.java | 6 +-
.../tajo/master/event/TaskRequestEvent.java | 9 +-
.../tajo/master/event/TaskTAttemptEvent.java | 10 +-
.../querymaster/QueryMasterManagerService.java | 22 +-
.../master/querymaster/QueryMasterTask.java | 14 +-
.../tajo/master/querymaster/QueryUnit.java | 907 -------------------
.../master/querymaster/QueryUnitAttempt.java | 443 ---------
.../tajo/master/querymaster/Repartitioner.java | 14 +-
.../tajo/master/querymaster/SubQuery.java | 72 +-
.../apache/tajo/master/querymaster/Task.java | 907 +++++++++++++++++++
.../tajo/master/querymaster/TaskAttempt.java | 443 +++++++++
.../main/java/org/apache/tajo/util/JSPUtil.java | 84 +-
.../apache/tajo/util/history/HistoryReader.java | 22 +-
.../apache/tajo/util/history/HistoryWriter.java | 4 +-
.../tajo/util/history/QueryUnitHistory.java | 167 ----
.../tajo/util/history/SubQueryHistory.java | 22 +-
.../apache/tajo/util/history/TaskHistory.java | 167 ++++
.../tajo/worker/ExecutionBlockContext.java | 12 +-
.../java/org/apache/tajo/worker/FetchImpl.java | 19 +-
.../apache/tajo/worker/InterDataRetriever.java | 8 +-
.../tajo/worker/TajoWorkerManagerService.java | 8 +-
.../main/java/org/apache/tajo/worker/Task.java | 31 +-
.../apache/tajo/worker/TaskAttemptContext.java | 12 +-
.../org/apache/tajo/worker/TaskHistory.java | 18 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 18 +-
.../apache/tajo/worker/TaskRunnerHistory.java | 16 +-
.../apache/tajo/worker/TaskRunnerManager.java | 10 +-
.../retriever/AdvancedDataRetriever.java | 14 +-
.../src/main/proto/QueryMasterProtocol.proto | 2 +-
.../src/main/proto/TajoWorkerProtocol.proto | 22 +-
.../main/resources/webapps/admin/querytasks.jsp | 70 +-
.../main/resources/webapps/admin/queryunit.jsp | 134 ---
.../src/main/resources/webapps/admin/task.jsp | 134 +++
.../resources/webapps/worker/querytasks.jsp | 70 +-
.../main/resources/webapps/worker/queryunit.jsp | 175 ----
.../src/main/resources/webapps/worker/task.jsp | 174 ++++
.../resources/webapps/worker/taskdetail.jsp | 14 +-
.../resources/webapps/worker/taskhistory.jsp | 6 +-
.../src/main/resources/webapps/worker/tasks.jsp | 14 +-
.../apache/tajo/LocalTajoTestingUtility.java | 10 +-
.../org/apache/tajo/TestQueryIdFactory.java | 6 +-
.../test/java/org/apache/tajo/TestTajoIds.java | 32 +-
.../org/apache/tajo/client/TestTajoClient.java | 12 +-
.../planner/physical/TestBNLJoinExec.java | 4 +-
.../planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 2 +-
.../physical/TestFullOuterHashJoinExec.java | 8 +-
.../physical/TestFullOuterMergeJoinExec.java | 12 +-
.../planner/physical/TestHashAntiJoinExec.java | 2 +-
.../planner/physical/TestHashJoinExec.java | 4 +-
.../planner/physical/TestHashSemiJoinExec.java | 2 +-
.../physical/TestLeftOuterHashJoinExec.java | 10 +-
.../physical/TestLeftOuterNLJoinExec.java | 10 +-
.../planner/physical/TestMergeJoinExec.java | 2 +-
.../engine/planner/physical/TestNLJoinExec.java | 4 +-
.../planner/physical/TestPhysicalPlanner.java | 60 +-
.../physical/TestProgressExternalSortExec.java | 2 +-
.../physical/TestRightOuterHashJoinExec.java | 6 +-
.../physical/TestRightOuterMergeJoinExec.java | 12 +-
.../engine/planner/physical/TestSortExec.java | 2 +-
.../tajo/engine/query/TestGroupByQuery.java | 4 +-
.../apache/tajo/master/TestRepartitioner.java | 21 +-
.../querymaster/TestIntermediateEntry.java | 2 +-
.../querymaster/TestQueryUnitStatusUpdate.java | 194 ----
.../querymaster/TestTaskStatusUpdate.java | 194 ++++
.../java/org/apache/tajo/util/TestJSPUtil.java | 54 +-
.../util/history/TestHistoryWriterReader.java | 39 +-
.../org/apache/tajo/worker/TestHistory.java | 8 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 4 +-
.../worker/dataserver/TestHttpDataServer.java | 12 +-
.../queries/TestQueryUnitStatusUpdate/case1.sql | 1 -
.../queries/TestQueryUnitStatusUpdate/case2.sql | 5 -
.../queries/TestQueryUnitStatusUpdate/case3.sql | 10 -
.../queries/TestTaskStatusUpdate/case1.sql | 1 +
.../queries/TestTaskStatusUpdate/case2.sql | 5 +
.../queries/TestTaskStatusUpdate/case3.sql | 10 +
tajo-dist/pom.xml | 4 +-
.../tajo/pullserver/HttpDataServerHandler.java | 2 +-
.../retriever/AdvancedDataRetriever.java | 16 +-
.../org/apache/tajo/storage/StorageManager.java | 6 +-
.../org/apache/tajo/storage/StorageUtil.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 6 +-
.../tajo/storage/hbase/HBasePutAppender.java | 4 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 2 +-
.../tajo/storage/hbase/HFileAppender.java | 8 +-
.../java/org/apache/tajo/storage/CSVFile.java | 4 +-
.../org/apache/tajo/storage/FileAppender.java | 6 +-
.../apache/tajo/storage/FileStorageManager.java | 8 +-
.../tajo/storage/HashShuffleAppender.java | 12 +-
.../storage/HashShuffleAppenderManager.java | 6 +-
.../java/org/apache/tajo/storage/RawFile.java | 4 +-
.../java/org/apache/tajo/storage/RowFile.java | 4 +-
.../apache/tajo/storage/avro/AvroAppender.java | 4 +-
.../tajo/storage/parquet/ParquetAppender.java | 4 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 4 +-
.../sequencefile/SequenceFileAppender.java | 4 +-
.../tajo/storage/text/DelimitedTextFile.java | 5 +-
128 files changed, 3390 insertions(+), 3397 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6b9146a..2230d0f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,7 +24,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
- TAJO-1221: HA TajoClient should not connect TajoMaster at the first. (jaehwa)
+ TAJO-1221: HA TajoClient should not connect TajoMaster at the first.
+ (jaehwa)
TAJO-1241: Change default client and table time zone behavior. (hyunsik)
@@ -224,6 +225,8 @@ Release 0.9.1 - unreleased
SUB TASKS
+ TAJO-324: Rename the prefix 'QueryUnit' to Task. (hyunsik)
+
TAJO-1151: Implement the ByteBuffer-based De/Serializer. (jinho)
TAJO-1152: RawFile ByteBuffer should be reuse. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java b/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java
index 8ac0d54..9599007 100644
--- a/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java
+++ b/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java
@@ -92,15 +92,15 @@ public class QueryIdFactory {
return new ExecutionBlockId(queryId, id);
}
- public synchronized static QueryUnitId newQueryUnitId(ExecutionBlockId executionBlockId) {
- return new QueryUnitId(executionBlockId, nextId.incrementAndGet());
+ public synchronized static TaskId newTaskId(ExecutionBlockId executionBlockId) {
+ return new TaskId(executionBlockId, nextId.incrementAndGet());
}
- public synchronized static QueryUnitId newQueryUnitId(ExecutionBlockId executionBlockId, int id) {
- return new QueryUnitId(executionBlockId, id);
+ public synchronized static TaskId newTaskId(ExecutionBlockId executionBlockId, int id) {
+ return new TaskId(executionBlockId, id);
}
- public synchronized static QueryUnitAttemptId newQueryUnitAttemptId(QueryUnitId queryUnitId, final int attemptId) {
- return new QueryUnitAttemptId(queryUnitId, attemptId);
+ public synchronized static TaskAttemptId newTaskAttemptId(TaskId taskId, final int attemptId) {
+ return new TaskAttemptId(taskId, attemptId);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/QueryUnitAttemptId.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/QueryUnitAttemptId.java b/tajo-common/src/main/java/org/apache/tajo/QueryUnitAttemptId.java
deleted file mode 100644
index a9fd68b..0000000
--- a/tajo-common/src/main/java/org/apache/tajo/QueryUnitAttemptId.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo;
-
-import com.google.common.base.Objects;
-
-public class QueryUnitAttemptId implements Comparable<QueryUnitAttemptId> {
- public static final String QUA_ID_PREFIX = "ta";
-
- private QueryUnitId queryUnitId;
- private int id;
-
- public QueryUnitId getQueryUnitId() {
- return queryUnitId;
- }
-
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
- public QueryUnitAttemptId(QueryUnitId queryUnitId, int id) {
- this.queryUnitId = queryUnitId;
- this.id = id;
- }
-
- public QueryUnitAttemptId(TajoIdProtos.QueryUnitAttemptIdProto proto) {
- this(new QueryUnitId(proto.getQueryUnitId()), proto.getId());
- }
-
- public TajoIdProtos.QueryUnitAttemptIdProto getProto() {
- return TajoIdProtos.QueryUnitAttemptIdProto.newBuilder()
- .setQueryUnitId(queryUnitId.getProto())
- .setId(id)
- .build();
- }
-
- @Override
- public int compareTo(QueryUnitAttemptId queryUnitAttemptId) {
- int result = queryUnitId.compareTo(queryUnitAttemptId.queryUnitId);
- if (result == 0) {
- return id - queryUnitAttemptId.id;
- } else {
- return result;
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (this == obj) {
- return true;
- }
- if(!(obj instanceof QueryUnitAttemptId)) {
- return false;
- }
- return compareTo((QueryUnitAttemptId)obj) == 0;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(queryUnitId, id);
- }
-
- @Override
- public String toString() {
- return QUA_ID_PREFIX + QueryId.SEPARATOR + toStringNoPrefix();
- }
-
- public String toStringNoPrefix() {
- return queryUnitId.toStringNoPrefix() + QueryId.SEPARATOR + QueryIdFactory.ATTEMPT_ID_FORMAT.format(id);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/QueryUnitId.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/QueryUnitId.java b/tajo-common/src/main/java/org/apache/tajo/QueryUnitId.java
deleted file mode 100644
index da0479b..0000000
--- a/tajo-common/src/main/java/org/apache/tajo/QueryUnitId.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo;
-
-import com.google.common.base.Objects;
-
-public class QueryUnitId implements Comparable<QueryUnitId> {
- public static final String QU_ID_PREFIX = "t";
-
- private ExecutionBlockId executionBlockId;
- private int id;
-
- public QueryUnitId(ExecutionBlockId executionBlockId, int id) {
- this.executionBlockId = executionBlockId;
- this.id = id;
- }
-
- public QueryUnitId(TajoIdProtos.QueryUnitIdProto proto) {
- this(new ExecutionBlockId(proto.getExecutionBlockId()), proto.getId());
- }
-
- public ExecutionBlockId getExecutionBlockId() {
- return executionBlockId;
- }
-
- public int getId() {
- return id;
- }
-
- public TajoIdProtos.QueryUnitIdProto getProto() {
- return TajoIdProtos.QueryUnitIdProto.newBuilder()
- .setExecutionBlockId(executionBlockId.getProto())
- .setId(id)
- .build();
- }
-
- @Override
- public int compareTo(QueryUnitId queryUnitId) {
- int result = executionBlockId.compareTo(queryUnitId.executionBlockId);
- if (result == 0) {
- return id - queryUnitId.id;
- } else {
- return result;
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof QueryUnitId)) {
- return false;
- }
- return compareTo((QueryUnitId) obj) == 0;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(executionBlockId, id);
- }
-
- @Override
- public String toString() {
- return QU_ID_PREFIX + QueryId.SEPARATOR + toStringNoPrefix();
- }
-
- public String toStringNoPrefix() {
- return executionBlockId.toStringNoPrefix() + QueryId.SEPARATOR + QueryIdFactory.QU_ID_FORMAT.format(id);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/TaskAttemptId.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/TaskAttemptId.java b/tajo-common/src/main/java/org/apache/tajo/TaskAttemptId.java
new file mode 100644
index 0000000..78c6325
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/TaskAttemptId.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Objects;
+
+public class TaskAttemptId implements Comparable<TaskAttemptId> {
+ public static final String QUA_ID_PREFIX = "ta";
+
+ private TaskId taskId;
+ private int id;
+
+ public TaskId getTaskId() {
+ return taskId;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public void setId(int id) {
+ this.id = id;
+ }
+
+ public TaskAttemptId(TaskId taskId, int id) {
+ this.taskId = taskId;
+ this.id = id;
+ }
+
+ public TaskAttemptId(TajoIdProtos.TaskAttemptIdProto proto) {
+ this(new TaskId(proto.getTaskId()), proto.getId());
+ }
+
+ public TajoIdProtos.TaskAttemptIdProto getProto() {
+ return TajoIdProtos.TaskAttemptIdProto.newBuilder()
+ .setTaskId(taskId.getProto())
+ .setId(id)
+ .build();
+ }
+
+ @Override
+ public int compareTo(TaskAttemptId taskAttemptId) {
+ int result = taskId.compareTo(taskAttemptId.taskId);
+ if (result == 0) {
+ return id - taskAttemptId.id;
+ } else {
+ return result;
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ if(!(obj instanceof TaskAttemptId)) {
+ return false;
+ }
+ return compareTo((TaskAttemptId)obj) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(taskId, id);
+ }
+
+ @Override
+ public String toString() {
+ return QUA_ID_PREFIX + QueryId.SEPARATOR + toStringNoPrefix();
+ }
+
+ public String toStringNoPrefix() {
+ return taskId.toStringNoPrefix() + QueryId.SEPARATOR + QueryIdFactory.ATTEMPT_ID_FORMAT.format(id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/TaskId.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/TaskId.java b/tajo-common/src/main/java/org/apache/tajo/TaskId.java
new file mode 100644
index 0000000..e1db67d
--- /dev/null
+++ b/tajo-common/src/main/java/org/apache/tajo/TaskId.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Objects;
+
+public class TaskId implements Comparable<TaskId> {
+ public static final String QU_ID_PREFIX = "t";
+
+ private ExecutionBlockId executionBlockId;
+ private int id;
+
+ public TaskId(ExecutionBlockId executionBlockId, int id) {
+ this.executionBlockId = executionBlockId;
+ this.id = id;
+ }
+
+ public TaskId(TajoIdProtos.TaskIdProto proto) {
+ this(new ExecutionBlockId(proto.getExecutionBlockId()), proto.getId());
+ }
+
+ public ExecutionBlockId getExecutionBlockId() {
+ return executionBlockId;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public TajoIdProtos.TaskIdProto getProto() {
+ return TajoIdProtos.TaskIdProto.newBuilder()
+ .setExecutionBlockId(executionBlockId.getProto())
+ .setId(id)
+ .build();
+ }
+
+ @Override
+ public int compareTo(TaskId taskId) {
+ int result = executionBlockId.compareTo(taskId.executionBlockId);
+ if (result == 0) {
+ return id - taskId.id;
+ } else {
+ return result;
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TaskId)) {
+ return false;
+ }
+ return compareTo((TaskId) obj) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(executionBlockId, id);
+ }
+
+ @Override
+ public String toString() {
+ return QU_ID_PREFIX + QueryId.SEPARATOR + toStringNoPrefix();
+ }
+
+ public String toStringNoPrefix() {
+ return executionBlockId.toStringNoPrefix() + QueryId.SEPARATOR + QueryIdFactory.QU_ID_FORMAT.format(id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/java/org/apache/tajo/util/TajoIdUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TajoIdUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/TajoIdUtils.java
index 978af6f..cc0f854 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TajoIdUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TajoIdUtils.java
@@ -20,8 +20,8 @@ package org.apache.tajo.util;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TaskId;
import java.text.DecimalFormat;
@@ -34,10 +34,10 @@ public class TajoIdUtils {
return new ExecutionBlockId(new QueryId(tokens[1], Integer.parseInt(tokens[2])), Integer.parseInt(tokens[3]));
}
- public static QueryUnitAttemptId parseQueryUnitAttemptId(String idStr) {
+ public static TaskAttemptId parseTaskAttemptId(String idStr) {
String[] tokens = idStr.split("_");
- return new QueryUnitAttemptId(new QueryUnitId(
+ return new TaskAttemptId(new TaskId(
new ExecutionBlockId(new QueryId(tokens[1], Integer.parseInt(tokens[2])), Integer.parseInt(tokens[3])),
Integer.parseInt(tokens[4])), Integer.parseInt(tokens[5]));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-common/src/main/proto/TajoIdProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/proto/TajoIdProtos.proto b/tajo-common/src/main/proto/TajoIdProtos.proto
index 1fb8bbd..c5e80a6 100644
--- a/tajo-common/src/main/proto/TajoIdProtos.proto
+++ b/tajo-common/src/main/proto/TajoIdProtos.proto
@@ -31,13 +31,13 @@ message ExecutionBlockIdProto {
required int32 id = 2;
}
-message QueryUnitIdProto {
+message TaskIdProto {
required ExecutionBlockIdProto executionBlockId = 1;
required int32 id = 2;
}
-message QueryUnitAttemptIdProto {
- required QueryUnitIdProto queryUnitId = 1;
+message TaskAttemptIdProto {
+ required TaskIdProto taskId = 1;
required int32 id = 2;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
index d051fb6..3c4949f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
@@ -27,7 +27,6 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.plan.logical.ShuffleFileWriteNode;
-import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.HashShuffleAppender;
import org.apache.tajo.storage.HashShuffleAppenderManager;
import org.apache.tajo.storage.Tuple;
@@ -87,7 +86,7 @@ public final class HashShuffleFileWriteExec extends UnaryPhysicalExec {
HashShuffleAppender appender = appenderMap.get(partId);
if (appender == null) {
appender = hashShuffleAppenderManager.getAppender(context.getConf(),
- context.getQueryId().getQueryUnitId().getExecutionBlockId(), partId, meta, outSchema);
+ context.getQueryId().getTaskId().getExecutionBlockId(), partId, meta, outSchema);
appenderMap.put(partId, appender);
}
return appender;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExec.java
index 67fb29b..de14c9a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExec.java
@@ -79,7 +79,7 @@ public abstract class PhysicalExec implements SchemaObject {
}
protected Path getExecutorTmpDir() {
- return new Path(context.getQueryId().getQueryUnitId().getExecutionBlockId().getQueryId().toString(),
+ return new Path(context.getQueryId().getTaskId().getExecutionBlockId().getQueryId().toString(),
UUID.randomUUID().toString());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index f507988..94cd4ed 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -88,7 +88,7 @@ public class SeqScanExec extends PhysicalExec {
}
cacheKey = new TupleCacheKey(
- context.getTaskId().getQueryUnitId().getExecutionBlockId().toString(), plan.getTableName(), pathNameKey);
+ context.getTaskId().getTaskId().getExecutionBlockId().toString(), plan.getTableName(), pathNameKey);
}
if (fragments != null
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java
deleted file mode 100644
index 3b0d60d..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- *
- */
-package org.apache.tajo.engine.query;
-
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.common.ProtoObject;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.planner.enforce.Enforcer;
-import org.apache.tajo.engine.planner.global.DataChannel;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.worker.FetchImpl;
-
-import java.util.List;
-
-public interface QueryUnitRequest extends ProtoObject<TajoWorkerProtocol.QueryUnitRequestProto> {
-
- public QueryUnitAttemptId getId();
- public List<CatalogProtos.FragmentProto> getFragments();
- public String getOutputTableId();
- public boolean isClusteredOutput();
- public String getSerializedData();
- public boolean isInterQuery();
- public void setInterQuery();
- public void addFetch(String name, FetchImpl fetch);
- public List<FetchImpl> getFetches();
- public boolean shouldDie();
- public void setShouldDie();
- public QueryContext getQueryContext(TajoConf conf);
- public DataChannel getDataChannel();
- public Enforcer getEnforcer();
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
deleted file mode 100644
index 1b89afd..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.engine.query;
-
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.engine.planner.enforce.Enforcer;
-import org.apache.tajo.engine.planner.global.DataChannel;
-import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProto;
-import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProtoOrBuilder;
-import org.apache.tajo.worker.FetchImpl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
-
-public class QueryUnitRequestImpl implements QueryUnitRequest {
-
- private QueryUnitAttemptId id;
- private List<FragmentProto> fragments;
- private String outputTable;
- private boolean isUpdated;
- private boolean clusteredOutput;
- private String serializedData; // logical node
- private Boolean interQuery;
- private List<FetchImpl> fetches;
- private Boolean shouldDie;
- private QueryContext queryContext;
- private DataChannel dataChannel;
- private Enforcer enforcer;
-
- private QueryUnitRequestProto proto = QueryUnitRequestProto.getDefaultInstance();
- private QueryUnitRequestProto.Builder builder = null;
- private boolean viaProto = false;
-
- public QueryUnitRequestImpl() {
- builder = QueryUnitRequestProto.newBuilder();
- this.id = null;
- this.isUpdated = false;
- }
-
- public QueryUnitRequestImpl(QueryUnitAttemptId id, List<FragmentProto> fragments,
- String outputTable, boolean clusteredOutput,
- String serializedData, QueryContext queryContext, DataChannel channel, Enforcer enforcer) {
- this();
- this.set(id, fragments, outputTable, clusteredOutput, serializedData, queryContext, channel, enforcer);
- }
-
- public QueryUnitRequestImpl(QueryUnitRequestProto proto) {
- this.proto = proto;
- viaProto = true;
- id = null;
- isUpdated = false;
- }
-
- public void set(QueryUnitAttemptId id, List<FragmentProto> fragments,
- String outputTable, boolean clusteredOutput,
- String serializedData, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) {
- this.id = id;
- this.fragments = fragments;
- this.outputTable = outputTable;
- this.clusteredOutput = clusteredOutput;
- this.serializedData = serializedData;
- this.isUpdated = true;
- this.queryContext = queryContext;
- this.queryContext = queryContext;
- this.dataChannel = dataChannel;
- this.enforcer = enforcer;
- }
-
- @Override
- public QueryUnitRequestProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public QueryUnitAttemptId getId() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (id != null) {
- return this.id;
- }
- if (!p.hasId()) {
- return null;
- }
- this.id = new QueryUnitAttemptId(p.getId());
- return this.id;
- }
-
- @Override
- public List<FragmentProto> getFragments() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (fragments != null) {
- return fragments;
- }
- if (fragments == null) {
- fragments = new ArrayList<FragmentProto>();
- }
- for (int i = 0; i < p.getFragmentsCount(); i++) {
- fragments.add(p.getFragments(i));
- }
- return this.fragments;
- }
-
- @Override
- public String getOutputTableId() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (outputTable != null) {
- return this.outputTable;
- }
- if (!p.hasOutputTable()) {
- return null;
- }
- this.outputTable = p.getOutputTable();
- return this.outputTable;
- }
-
- @Override
- public boolean isClusteredOutput() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (isUpdated) {
- return this.clusteredOutput;
- }
- if (!p.hasClusteredOutput()) {
- return false;
- }
- this.clusteredOutput = p.getClusteredOutput();
- this.isUpdated = true;
- return this.clusteredOutput;
- }
-
- @Override
- public String getSerializedData() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.serializedData != null) {
- return this.serializedData;
- }
- if (!p.hasSerializedData()) {
- return null;
- }
- this.serializedData = p.getSerializedData();
- return this.serializedData;
- }
-
- public boolean isInterQuery() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (interQuery != null) {
- return interQuery;
- }
- if (!p.hasInterQuery()) {
- return false;
- }
- this.interQuery = p.getInterQuery();
- return this.interQuery;
- }
-
- public void setInterQuery() {
- maybeInitBuilder();
- this.interQuery = true;
- }
-
- public void addFetch(String name, FetchImpl fetch) {
- maybeInitBuilder();
- initFetches();
- fetch.setName(name);
- fetches.add(fetch);
- }
-
- public QueryContext getQueryContext(TajoConf conf) {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (queryContext != null) {
- return queryContext;
- }
- if (!p.hasQueryContext()) {
- return null;
- }
- this.queryContext = new QueryContext(conf, p.getQueryContext());
- return this.queryContext;
- }
-
- public void setQueryContext(QueryContext queryContext) {
- maybeInitBuilder();
- this.queryContext = queryContext;
- }
-
- public void setDataChannel(DataChannel dataChannel) {
- maybeInitBuilder();
- this.dataChannel = dataChannel;
- }
-
- @Override
- public DataChannel getDataChannel() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (dataChannel != null) {
- return dataChannel;
- }
- if (!p.hasDataChannel()) {
- return null;
- }
- this.dataChannel = new DataChannel(p.getDataChannel());
- return this.dataChannel;
- }
-
- @Override
- public Enforcer getEnforcer() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (enforcer != null) {
- return enforcer;
- }
- if (!p.hasEnforcer()) {
- return null;
- }
- this.enforcer = new Enforcer(p.getEnforcer());
- return this.enforcer;
- }
-
- public List<FetchImpl> getFetches() {
- initFetches();
-
- return this.fetches;
- }
-
- private void initFetches() {
- if (this.fetches != null) {
- return;
- }
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- this.fetches = new ArrayList<FetchImpl>();
- for(TajoWorkerProtocol.FetchProto fetch : p.getFetchesList()) {
- fetches.add(new FetchImpl(fetch));
- }
- }
-
- @Override
- public boolean shouldDie() {
- QueryUnitRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (shouldDie != null) {
- return shouldDie;
- }
- if (!p.hasShouldDie()) {
- return false;
- }
- this.shouldDie = p.getShouldDie();
- return this.shouldDie;
- }
-
- @Override
- public void setShouldDie() {
- maybeInitBuilder();
- shouldDie = true;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = QueryUnitRequestProto.newBuilder(proto);
- }
- viaProto = true;
- }
-
- private void mergeLocalToBuilder() {
- if (id != null) {
- builder.setId(this.id.getProto());
- }
- if (fragments != null) {
- for (int i = 0; i < fragments.size(); i++) {
- builder.addFragments(fragments.get(i));
- }
- }
- if (this.outputTable != null) {
- builder.setOutputTable(this.outputTable);
- }
- if (this.isUpdated) {
- builder.setClusteredOutput(this.clusteredOutput);
- }
- if (this.serializedData != null) {
- builder.setSerializedData(this.serializedData);
- }
- if (this.interQuery != null) {
- builder.setInterQuery(this.interQuery);
- }
- if (this.fetches != null) {
- for (int i = 0; i < fetches.size(); i++) {
- builder.addFetches(fetches.get(i).getProto());
- }
- }
- if (this.shouldDie != null) {
- builder.setShouldDie(this.shouldDie);
- }
- if (this.queryContext != null) {
- builder.setQueryContext(queryContext.getProto());
- }
- if (this.dataChannel != null) {
- builder.setDataChannel(dataChannel.getProto());
- }
- if (this.enforcer != null) {
- builder.setEnforcer(enforcer.getProto());
- }
- }
-
- private void mergeLocalToProto() {
- if(viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
new file mode 100644
index 0000000..a3e586a
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequest.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.planner.global.DataChannel;
+import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.worker.FetchImpl;
+
+import java.util.List;
+
+public interface TaskRequest extends ProtoObject<TajoWorkerProtocol.TaskRequestProto> {
+
+ public TaskAttemptId getId();
+ public List<CatalogProtos.FragmentProto> getFragments();
+ public String getOutputTableId();
+ public boolean isClusteredOutput();
+ public String getSerializedData();
+ public boolean isInterQuery();
+ public void setInterQuery();
+ public void addFetch(String name, FetchImpl fetch);
+ public List<FetchImpl> getFetches();
+ public boolean shouldDie();
+ public void setShouldDie();
+ public QueryContext getQueryContext(TajoConf conf);
+ public DataChannel getDataChannel();
+ public Enforcer getEnforcer();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
new file mode 100644
index 0000000..cef5488
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.planner.global.DataChannel;
+import org.apache.tajo.ipc.TajoWorkerProtocol;
+import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProto;
+import org.apache.tajo.ipc.TajoWorkerProtocol.TaskRequestProtoOrBuilder;
+import org.apache.tajo.worker.FetchImpl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
+
+public class TaskRequestImpl implements TaskRequest {
+
+ private TaskAttemptId id;
+ private List<FragmentProto> fragments;
+ private String outputTable;
+ private boolean isUpdated;
+ private boolean clusteredOutput;
+ private String serializedData; // logical node
+ private Boolean interQuery;
+ private List<FetchImpl> fetches;
+ private Boolean shouldDie;
+ private QueryContext queryContext;
+ private DataChannel dataChannel;
+ private Enforcer enforcer;
+
+ private TaskRequestProto proto = TajoWorkerProtocol.TaskRequestProto.getDefaultInstance();
+ private TajoWorkerProtocol.TaskRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public TaskRequestImpl() {
+ builder = TaskRequestProto.newBuilder();
+ this.id = null;
+ this.isUpdated = false;
+ }
+
+ public TaskRequestImpl(TaskAttemptId id, List<FragmentProto> fragments,
+ String outputTable, boolean clusteredOutput,
+ String serializedData, QueryContext queryContext, DataChannel channel, Enforcer enforcer) {
+ this();
+ this.set(id, fragments, outputTable, clusteredOutput, serializedData, queryContext, channel, enforcer);
+ }
+
+ public TaskRequestImpl(TaskRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ id = null;
+ isUpdated = false;
+ }
+
+ public void set(TaskAttemptId id, List<FragmentProto> fragments,
+ String outputTable, boolean clusteredOutput,
+ String serializedData, QueryContext queryContext, DataChannel dataChannel, Enforcer enforcer) {
+ this.id = id;
+ this.fragments = fragments;
+ this.outputTable = outputTable;
+ this.clusteredOutput = clusteredOutput;
+ this.serializedData = serializedData;
+ this.isUpdated = true;
+ this.queryContext = queryContext;
+ this.queryContext = queryContext;
+ this.dataChannel = dataChannel;
+ this.enforcer = enforcer;
+ }
+
+ @Override
+ public TaskRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public TaskAttemptId getId() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (id != null) {
+ return this.id;
+ }
+ if (!p.hasId()) {
+ return null;
+ }
+ this.id = new TaskAttemptId(p.getId());
+ return this.id;
+ }
+
+ @Override
+ public List<FragmentProto> getFragments() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (fragments != null) {
+ return fragments;
+ }
+ if (fragments == null) {
+ fragments = new ArrayList<FragmentProto>();
+ }
+ for (int i = 0; i < p.getFragmentsCount(); i++) {
+ fragments.add(p.getFragments(i));
+ }
+ return this.fragments;
+ }
+
+ @Override
+ public String getOutputTableId() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (outputTable != null) {
+ return this.outputTable;
+ }
+ if (!p.hasOutputTable()) {
+ return null;
+ }
+ this.outputTable = p.getOutputTable();
+ return this.outputTable;
+ }
+
+ @Override
+ public boolean isClusteredOutput() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (isUpdated) {
+ return this.clusteredOutput;
+ }
+ if (!p.hasClusteredOutput()) {
+ return false;
+ }
+ this.clusteredOutput = p.getClusteredOutput();
+ this.isUpdated = true;
+ return this.clusteredOutput;
+ }
+
+ @Override
+ public String getSerializedData() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.serializedData != null) {
+ return this.serializedData;
+ }
+ if (!p.hasSerializedData()) {
+ return null;
+ }
+ this.serializedData = p.getSerializedData();
+ return this.serializedData;
+ }
+
+ public boolean isInterQuery() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (interQuery != null) {
+ return interQuery;
+ }
+ if (!p.hasInterQuery()) {
+ return false;
+ }
+ this.interQuery = p.getInterQuery();
+ return this.interQuery;
+ }
+
+ public void setInterQuery() {
+ maybeInitBuilder();
+ this.interQuery = true;
+ }
+
+ public void addFetch(String name, FetchImpl fetch) {
+ maybeInitBuilder();
+ initFetches();
+ fetch.setName(name);
+ fetches.add(fetch);
+ }
+
+ public QueryContext getQueryContext(TajoConf conf) {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (queryContext != null) {
+ return queryContext;
+ }
+ if (!p.hasQueryContext()) {
+ return null;
+ }
+ this.queryContext = new QueryContext(conf, p.getQueryContext());
+ return this.queryContext;
+ }
+
+ public void setQueryContext(QueryContext queryContext) {
+ maybeInitBuilder();
+ this.queryContext = queryContext;
+ }
+
+ public void setDataChannel(DataChannel dataChannel) {
+ maybeInitBuilder();
+ this.dataChannel = dataChannel;
+ }
+
+ @Override
+ public DataChannel getDataChannel() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (dataChannel != null) {
+ return dataChannel;
+ }
+ if (!p.hasDataChannel()) {
+ return null;
+ }
+ this.dataChannel = new DataChannel(p.getDataChannel());
+ return this.dataChannel;
+ }
+
+ @Override
+ public Enforcer getEnforcer() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (enforcer != null) {
+ return enforcer;
+ }
+ if (!p.hasEnforcer()) {
+ return null;
+ }
+ this.enforcer = new Enforcer(p.getEnforcer());
+ return this.enforcer;
+ }
+
+ public List<FetchImpl> getFetches() {
+ initFetches();
+
+ return this.fetches;
+ }
+
+ private void initFetches() {
+ if (this.fetches != null) {
+ return;
+ }
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ this.fetches = new ArrayList<FetchImpl>();
+ for(TajoWorkerProtocol.FetchProto fetch : p.getFetchesList()) {
+ fetches.add(new FetchImpl(fetch));
+ }
+ }
+
+ @Override
+ public boolean shouldDie() {
+ TaskRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (shouldDie != null) {
+ return shouldDie;
+ }
+ if (!p.hasShouldDie()) {
+ return false;
+ }
+ this.shouldDie = p.getShouldDie();
+ return this.shouldDie;
+ }
+
+ @Override
+ public void setShouldDie() {
+ maybeInitBuilder();
+ shouldDie = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = TajoWorkerProtocol.TaskRequestProto.newBuilder(proto);
+ }
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (id != null) {
+ builder.setId(this.id.getProto());
+ }
+ if (fragments != null) {
+ for (int i = 0; i < fragments.size(); i++) {
+ builder.addFragments(fragments.get(i));
+ }
+ }
+ if (this.outputTable != null) {
+ builder.setOutputTable(this.outputTable);
+ }
+ if (this.isUpdated) {
+ builder.setClusteredOutput(this.clusteredOutput);
+ }
+ if (this.serializedData != null) {
+ builder.setSerializedData(this.serializedData);
+ }
+ if (this.interQuery != null) {
+ builder.setInterQuery(this.interQuery);
+ }
+ if (this.fetches != null) {
+ for (int i = 0; i < fetches.size(); i++) {
+ builder.addFetches(fetches.get(i).getProto());
+ }
+ }
+ if (this.shouldDie != null) {
+ builder.setShouldDie(this.shouldDie);
+ }
+ if (this.queryContext != null) {
+ builder.setQueryContext(queryContext.getProto());
+ }
+ if (this.dataChannel != null) {
+ builder.setDataChannel(dataChannel.getProto());
+ }
+ if (this.enforcer != null) {
+ builder.setEnforcer(enforcer.getProto());
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if(viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
index 01137aa..d9d496e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
@@ -28,18 +28,18 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
-import org.apache.tajo.engine.query.QueryUnitRequest;
-import org.apache.tajo.engine.query.QueryUnitRequestImpl;
+import org.apache.tajo.engine.query.TaskRequest;
+import org.apache.tajo.engine.query.TaskRequestImpl;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
-import org.apache.tajo.master.querymaster.QueryUnit;
-import org.apache.tajo.master.querymaster.QueryUnitAttempt;
+import org.apache.tajo.master.querymaster.Task;
+import org.apache.tajo.master.querymaster.TaskAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.DataLocation;
@@ -114,14 +114,14 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
super.start();
}
- private static final QueryUnitAttemptId NULL_ATTEMPT_ID;
- public static final TajoWorkerProtocol.QueryUnitRequestProto stopTaskRunnerReq;
+ private static final TaskAttemptId NULL_ATTEMPT_ID;
+ public static final TajoWorkerProtocol.TaskRequestProto stopTaskRunnerReq;
static {
ExecutionBlockId nullSubQuery = QueryIdFactory.newExecutionBlockId(QueryIdFactory.NULL_QUERY_ID, 0);
- NULL_ATTEMPT_ID = QueryIdFactory.newQueryUnitAttemptId(QueryIdFactory.newQueryUnitId(nullSubQuery, 0), 0);
+ NULL_ATTEMPT_ID = QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(nullSubQuery, 0), 0);
- TajoWorkerProtocol.QueryUnitRequestProto.Builder builder =
- TajoWorkerProtocol.QueryUnitRequestProto.newBuilder();
+ TajoWorkerProtocol.TaskRequestProto.Builder builder =
+ TajoWorkerProtocol.TaskRequestProto.newBuilder();
builder.setId(NULL_ATTEMPT_ID.getProto());
builder.setShouldDie(true);
builder.setOutputTable("");
@@ -191,8 +191,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
if (event instanceof FragmentScheduleEvent) {
FragmentScheduleEvent castEvent = (FragmentScheduleEvent) event;
if (context.isLeafQuery()) {
- QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext();
- QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
+ TaskAttemptScheduleContext taskContext = new TaskAttemptScheduleContext();
+ Task task = SubQuery.newEmptyTask(context, taskContext, subQuery, nextTaskId++);
task.addFragment(castEvent.getLeftFragment(), true);
scheduledObjectNum++;
if (castEvent.hasRightFragments()) {
@@ -216,8 +216,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
} else if (event instanceof FetchScheduleEvent) {
FetchScheduleEvent castEvent = (FetchScheduleEvent) event;
Map<String, List<FetchImpl>> fetches = castEvent.getFetches();
- QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext();
- QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
+ TaskAttemptScheduleContext taskScheduleContext = new TaskAttemptScheduleContext();
+ Task task = SubQuery.newEmptyTask(context, taskScheduleContext, subQuery, nextTaskId++);
scheduledObjectNum++;
for (Entry<String, List<FetchImpl>> eachFetch : fetches.entrySet()) {
task.addFetches(eachFetch.getKey(), eachFetch.getValue());
@@ -230,8 +230,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
task.addFragments(Arrays.asList(broadcastFragmentsForNonLeafTask));
}
subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
- } else if (event instanceof QueryUnitAttemptScheduleEvent) {
- QueryUnitAttemptScheduleEvent castEvent = (QueryUnitAttemptScheduleEvent) event;
+ } else if (event instanceof TaskAttemptToSchedulerEvent) {
+ TaskAttemptToSchedulerEvent castEvent = (TaskAttemptToSchedulerEvent) event;
if (context.isLeafQuery()) {
scheduledRequests.addLeafTask(castEvent);
} else {
@@ -240,12 +240,12 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
} else if (event.getType() == EventType.T_SCHEDULE_CANCEL) {
// when a subquery is killed, unassigned query unit attmpts are canceled from the scheduler.
- // This event is triggered by QueryUnitAttempt.
- QueryUnitAttemptScheduleEvent castedEvent = (QueryUnitAttemptScheduleEvent) event;
- scheduledRequests.leafTasks.remove(castedEvent.getQueryUnitAttempt().getId());
- LOG.info(castedEvent.getQueryUnitAttempt().getId() + " is canceled from " + this.getClass().getSimpleName());
- ((QueryUnitAttemptScheduleEvent) event).getQueryUnitAttempt().handle(
- new TaskAttemptEvent(castedEvent.getQueryUnitAttempt().getId(), TaskAttemptEventType.TA_SCHEDULE_CANCELED));
+ // This event is triggered by TaskAttempt.
+ TaskAttemptToSchedulerEvent castedEvent = (TaskAttemptToSchedulerEvent) event;
+ scheduledRequests.leafTasks.remove(castedEvent.getTaskAttempt().getId());
+ LOG.info(castedEvent.getTaskAttempt().getId() + " is canceled from " + this.getClass().getSimpleName());
+ ((TaskAttemptToSchedulerEvent) event).getTaskAttempt().handle(
+ new TaskAttemptEvent(castedEvent.getTaskAttempt().getId(), TaskAttemptEventType.TA_SCHEDULE_CANCELED));
}
}
@@ -337,8 +337,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
private final String host;
private final String rack;
/** A key is disk volume, and a value is a list of tasks to be scheduled. */
- private Map<Integer, LinkedHashSet<QueryUnitAttempt>> unassignedTaskForEachVolume =
- Collections.synchronizedMap(new HashMap<Integer, LinkedHashSet<QueryUnitAttempt>>());
+ private Map<Integer, LinkedHashSet<TaskAttempt>> unassignedTaskForEachVolume =
+ Collections.synchronizedMap(new HashMap<Integer, LinkedHashSet<TaskAttempt>>());
/** A value is last assigned volume id for each task runner */
private HashMap<TajoContainerId, Integer> lastAssignedVolumeId = new HashMap<TajoContainerId,
Integer>();
@@ -360,11 +360,11 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
this.rack = rack;
}
- public synchronized void addQueryUnitAttempt(int volumeId, QueryUnitAttempt attemptId){
+ public synchronized void addTaskAttempt(int volumeId, TaskAttempt attemptId){
synchronized (unassignedTaskForEachVolume){
- LinkedHashSet<QueryUnitAttempt> list = unassignedTaskForEachVolume.get(volumeId);
+ LinkedHashSet<TaskAttempt> list = unassignedTaskForEachVolume.get(volumeId);
if (list == null) {
- list = new LinkedHashSet<QueryUnitAttempt>();
+ list = new LinkedHashSet<TaskAttempt>();
unassignedTaskForEachVolume.put(volumeId, list);
}
list.add(attemptId);
@@ -381,9 +381,9 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
* 2. unknown block or Non-splittable task in host
* 3. remote tasks. unassignedTaskForEachVolume is only contained local task. so it will be null
*/
- public synchronized QueryUnitAttemptId getLocalTask(TajoContainerId containerId) {
+ public synchronized TaskAttemptId getLocalTask(TajoContainerId containerId) {
int volumeId;
- QueryUnitAttemptId queryUnitAttemptId = null;
+ TaskAttemptId taskAttemptId = null;
if (!lastAssignedVolumeId.containsKey(containerId)) {
volumeId = getLowestVolumeId();
@@ -396,7 +396,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
int retry = unassignedTaskForEachVolume.size();
do {
//clean and get a remaining local task
- queryUnitAttemptId = getAndRemove(volumeId);
+ taskAttemptId = getAndRemove(volumeId);
if(!unassignedTaskForEachVolume.containsKey(volumeId)) {
decreaseConcurrency(containerId);
if (volumeId > REMOTE) {
@@ -404,7 +404,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- if (queryUnitAttemptId == null) {
+ if (taskAttemptId == null) {
//reassign next volume
volumeId = getLowestVolumeId();
increaseConcurrency(containerId, volumeId);
@@ -416,19 +416,19 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
} else {
this.remainTasksNum.set(0);
}
- return queryUnitAttemptId;
+ return taskAttemptId;
}
- public synchronized QueryUnitAttemptId getQueryUnitAttemptIdByRack(String rack) {
- QueryUnitAttemptId queryUnitAttemptId = null;
+ public synchronized TaskAttemptId getTaskAttemptIdByRack(String rack) {
+ TaskAttemptId taskAttemptId = null;
if (unassignedTaskForEachVolume.size() > 0 && this.rack.equals(rack)) {
int retry = unassignedTaskForEachVolume.size();
do {
//clean and get a remaining task
int volumeId = getLowestVolumeId();
- queryUnitAttemptId = getAndRemove(volumeId);
- if (queryUnitAttemptId == null) {
+ taskAttemptId = getAndRemove(volumeId);
+ if (taskAttemptId == null) {
if (volumeId > REMOTE) {
diskVolumeLoads.remove(volumeId);
}
@@ -438,29 +438,29 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
} while (retry > 0);
}
- return queryUnitAttemptId;
+ return taskAttemptId;
}
- private synchronized QueryUnitAttemptId getAndRemove(int volumeId){
- QueryUnitAttemptId queryUnitAttemptId = null;
- if(!unassignedTaskForEachVolume.containsKey(volumeId)) return queryUnitAttemptId;
+ private synchronized TaskAttemptId getAndRemove(int volumeId){
+ TaskAttemptId taskAttemptId = null;
+ if(!unassignedTaskForEachVolume.containsKey(volumeId)) return taskAttemptId;
- LinkedHashSet<QueryUnitAttempt> list = unassignedTaskForEachVolume.get(volumeId);
+ LinkedHashSet<TaskAttempt> list = unassignedTaskForEachVolume.get(volumeId);
if(list != null && list.size() > 0){
- QueryUnitAttempt queryUnitAttempt;
+ TaskAttempt taskAttempt;
synchronized (unassignedTaskForEachVolume) {
- Iterator<QueryUnitAttempt> iterator = list.iterator();
- queryUnitAttempt = iterator.next();
+ Iterator<TaskAttempt> iterator = list.iterator();
+ taskAttempt = iterator.next();
iterator.remove();
}
this.remainTasksNum.getAndDecrement();
- queryUnitAttemptId = queryUnitAttempt.getId();
- for (DataLocation location : queryUnitAttempt.getQueryUnit().getDataLocations()) {
+ taskAttemptId = taskAttempt.getId();
+ for (DataLocation location : taskAttempt.getTask().getDataLocations()) {
if (!this.getHost().equals(location.getHost())) {
HostVolumeMapping volumeMapping = scheduledRequests.leafTaskHostMapping.get(location.getHost());
if (volumeMapping != null) {
- volumeMapping.removeQueryUnitAttempt(location.getVolumeId(), queryUnitAttempt);
+ volumeMapping.removeTaskAttempt(location.getVolumeId(), taskAttempt);
}
}
}
@@ -469,16 +469,16 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
if(list == null || list.isEmpty()) {
unassignedTaskForEachVolume.remove(volumeId);
}
- return queryUnitAttemptId;
+ return taskAttemptId;
}
- private synchronized void removeQueryUnitAttempt(int volumeId, QueryUnitAttempt queryUnitAttempt){
+ private synchronized void removeTaskAttempt(int volumeId, TaskAttempt taskAttempt){
if(!unassignedTaskForEachVolume.containsKey(volumeId)) return;
- LinkedHashSet<QueryUnitAttempt> tasks = unassignedTaskForEachVolume.get(volumeId);
+ LinkedHashSet<TaskAttempt> tasks = unassignedTaskForEachVolume.get(volumeId);
if(tasks != null && tasks.size() > 0){
- tasks.remove(queryUnitAttempt);
+ tasks.remove(taskAttempt);
remainTasksNum.getAndDecrement();
} else {
unassignedTaskForEachVolume.remove(volumeId);
@@ -596,14 +596,14 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
// two list leafTasks and nonLeafTasks keep all tasks to be scheduled. Even though some task is included in
// leafTaskHostMapping or leafTasksRackMapping, some task T will not be sent to a task runner
// if the task is not included in leafTasks and nonLeafTasks.
- private final Set<QueryUnitAttemptId> leafTasks = Collections.synchronizedSet(new HashSet<QueryUnitAttemptId>());
- private final Set<QueryUnitAttemptId> nonLeafTasks = Collections.synchronizedSet(new HashSet<QueryUnitAttemptId>());
+ private final Set<TaskAttemptId> leafTasks = Collections.synchronizedSet(new HashSet<TaskAttemptId>());
+ private final Set<TaskAttemptId> nonLeafTasks = Collections.synchronizedSet(new HashSet<TaskAttemptId>());
private Map<String, HostVolumeMapping> leafTaskHostMapping = Maps.newConcurrentMap();
- private final Map<String, HashSet<QueryUnitAttemptId>> leafTasksRackMapping = Maps.newConcurrentMap();
+ private final Map<String, HashSet<TaskAttemptId>> leafTasksRackMapping = Maps.newConcurrentMap();
- private synchronized void addLeafTask(QueryUnitAttemptScheduleEvent event) {
- QueryUnitAttempt queryUnitAttempt = event.getQueryUnitAttempt();
- List<DataLocation> locations = queryUnitAttempt.getQueryUnit().getDataLocations();
+ private synchronized void addLeafTask(TaskAttemptToSchedulerEvent event) {
+ TaskAttempt taskAttempt = event.getTaskAttempt();
+ List<DataLocation> locations = taskAttempt.getTask().getDataLocations();
for (DataLocation location : locations) {
String host = location.getHost();
@@ -614,30 +614,30 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
hostVolumeMapping = new HostVolumeMapping(host, rack);
leafTaskHostMapping.put(host, hostVolumeMapping);
}
- hostVolumeMapping.addQueryUnitAttempt(location.getVolumeId(), queryUnitAttempt);
+ hostVolumeMapping.addTaskAttempt(location.getVolumeId(), taskAttempt);
if (LOG.isDebugEnabled()) {
LOG.debug("Added attempt req to host " + host);
}
- HashSet<QueryUnitAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack());
+ HashSet<TaskAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack());
if (list == null) {
- list = new HashSet<QueryUnitAttemptId>();
+ list = new HashSet<TaskAttemptId>();
leafTasksRackMapping.put(hostVolumeMapping.getRack(), list);
}
- list.add(queryUnitAttempt.getId());
+ list.add(taskAttempt.getId());
if (LOG.isDebugEnabled()) {
LOG.debug("Added attempt req to rack " + hostVolumeMapping.getRack());
}
}
- leafTasks.add(queryUnitAttempt.getId());
+ leafTasks.add(taskAttempt.getId());
}
- private void addNonLeafTask(QueryUnitAttemptScheduleEvent event) {
- nonLeafTasks.add(event.getQueryUnitAttempt().getId());
+ private void addNonLeafTask(TaskAttemptToSchedulerEvent event) {
+ nonLeafTasks.add(event.getTaskAttempt().getId());
}
public int leafTaskNum() {
@@ -648,14 +648,14 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
return nonLeafTasks.size();
}
- public Set<QueryUnitAttemptId> assignedRequest = new HashSet<QueryUnitAttemptId>();
+ public Set<TaskAttemptId> assignedRequest = new HashSet<TaskAttemptId>();
- private QueryUnitAttemptId allocateLocalTask(String host, TajoContainerId containerId){
+ private TaskAttemptId allocateLocalTask(String host, TajoContainerId containerId){
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
if (hostVolumeMapping != null) { //tajo host is located in hadoop datanode
for (int i = 0; i < hostVolumeMapping.getRemainingLocalTaskSize(); i++) {
- QueryUnitAttemptId attemptId = hostVolumeMapping.getLocalTask(containerId);
+ TaskAttemptId attemptId = hostVolumeMapping.getLocalTask(containerId);
if(attemptId == null) break;
//find remaining local task
@@ -671,11 +671,11 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
return null;
}
- private QueryUnitAttemptId allocateRackTask(String host) {
+ private TaskAttemptId allocateRackTask(String host) {
List<HostVolumeMapping> remainingTasks = Lists.newArrayList(leafTaskHostMapping.values());
String rack = RackResolver.resolve(host).getNetworkLocation();
- QueryUnitAttemptId attemptId = null;
+ TaskAttemptId attemptId = null;
if (remainingTasks.size() > 0) {
synchronized (scheduledRequests) {
@@ -697,7 +697,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
for (HostVolumeMapping tasks : remainingTasks) {
for (int i = 0; i < tasks.getRemainingLocalTaskSize(); i++) {
- QueryUnitAttemptId tId = tasks.getQueryUnitAttemptIdByRack(rack);
+ TaskAttemptId tId = tasks.getTaskAttemptIdByRack(rack);
if (tId == null) break;
@@ -713,12 +713,12 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
//find task in rack
if (attemptId == null) {
- HashSet<QueryUnitAttemptId> list = leafTasksRackMapping.get(rack);
+ HashSet<TaskAttemptId> list = leafTasksRackMapping.get(rack);
if (list != null) {
synchronized (list) {
- Iterator<QueryUnitAttemptId> iterator = list.iterator();
+ Iterator<TaskAttemptId> iterator = list.iterator();
while (iterator.hasNext()) {
- QueryUnitAttemptId tId = iterator.next();
+ TaskAttemptId tId = iterator.next();
iterator.remove();
if (leafTasks.contains(tId)) {
leafTasks.remove(tId);
@@ -788,7 +788,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
//////////////////////////////////////////////////////////////////////
// disk or host-local allocation
//////////////////////////////////////////////////////////////////////
- QueryUnitAttemptId attemptId = allocateLocalTask(host, containerId);
+ TaskAttemptId attemptId = allocateLocalTask(host, containerId);
if (attemptId == null) { // if a local task cannot be found
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
@@ -832,8 +832,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
if (attemptId != null) {
- QueryUnit task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
- QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
+ Task task = subQuery.getTask(attemptId.getTaskId());
+ TaskRequest taskAssign = new TaskRequestImpl(
attemptId,
new ArrayList<FragmentProto>(task.getAllFragments()),
"",
@@ -878,7 +878,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
taskRequest = taskRequests.pollFirst();
LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId());
- QueryUnitAttemptId attemptId;
+ TaskAttemptId attemptId;
// random allocation
if (nonLeafTasks.size() > 0) {
synchronized (nonLeafTasks){
@@ -887,9 +887,9 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
LOG.debug("Assigned based on * match");
- QueryUnit task;
- task = subQuery.getQueryUnit(attemptId.getQueryUnitId());
- QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
+ Task task;
+ task = subQuery.getTask(attemptId.getTaskId());
+ TaskRequest taskAssign = new TaskRequestImpl(
attemptId,
Lists.newArrayList(task.getAllFragments()),
"",
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
index cc99453..0ab19db 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
@@ -24,19 +24,19 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.global.ExecutionBlock;
import org.apache.tajo.engine.planner.global.MasterPlan;
-import org.apache.tajo.engine.query.QueryUnitRequest;
-import org.apache.tajo.engine.query.QueryUnitRequestImpl;
+import org.apache.tajo.engine.query.TaskRequest;
+import org.apache.tajo.engine.query.TaskRequestImpl;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.event.*;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext;
import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
-import org.apache.tajo.master.querymaster.QueryUnit;
-import org.apache.tajo.master.querymaster.QueryUnitAttempt;
+import org.apache.tajo.master.querymaster.Task;
+import org.apache.tajo.master.querymaster.TaskAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.StorageManager;
@@ -126,14 +126,14 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
super.start();
}
- private static final QueryUnitAttemptId NULL_ATTEMPT_ID;
- public static final TajoWorkerProtocol.QueryUnitRequestProto stopTaskRunnerReq;
+ private static final TaskAttemptId NULL_ATTEMPT_ID;
+ public static final TajoWorkerProtocol.TaskRequestProto stopTaskRunnerReq;
static {
ExecutionBlockId nullSubQuery = QueryIdFactory.newExecutionBlockId(QueryIdFactory.NULL_QUERY_ID, 0);
- NULL_ATTEMPT_ID = QueryIdFactory.newQueryUnitAttemptId(QueryIdFactory.newQueryUnitId(nullSubQuery, 0), 0);
+ NULL_ATTEMPT_ID = QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(nullSubQuery, 0), 0);
- TajoWorkerProtocol.QueryUnitRequestProto.Builder builder =
- TajoWorkerProtocol.QueryUnitRequestProto.newBuilder();
+ TajoWorkerProtocol.TaskRequestProto.Builder builder =
+ TajoWorkerProtocol.TaskRequestProto.newBuilder();
builder.setId(NULL_ATTEMPT_ID.getProto());
builder.setShouldDie(true);
builder.setOutputTable("");
@@ -214,9 +214,9 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
} else if (event instanceof FetchScheduleEvent) {
FetchScheduleEvent castEvent = (FetchScheduleEvent) event;
scheduledFetches.addFetch(castEvent.getFetches());
- } else if (event instanceof QueryUnitAttemptScheduleEvent) {
- QueryUnitAttemptScheduleEvent castEvent = (QueryUnitAttemptScheduleEvent) event;
- assignTask(castEvent.getContext(), castEvent.getQueryUnitAttempt());
+ } else if (event instanceof TaskAttemptToSchedulerEvent) {
+ TaskAttemptToSchedulerEvent castEvent = (TaskAttemptToSchedulerEvent) event;
+ assignTask(castEvent.getContext(), castEvent.getTaskAttempt());
}
}
}
@@ -360,9 +360,9 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
}
String host = container.getTaskHostName();
- QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext(container.containerID,
+ TaskAttemptScheduleContext taskContext = new TaskAttemptScheduleContext(container.containerID,
host, taskRequest.getCallback());
- QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
+ Task task = SubQuery.newEmptyTask(context, taskContext, subQuery, nextTaskId++);
FragmentPair fragmentPair;
List<FragmentPair> fragmentPairs = new ArrayList<FragmentPair>();
@@ -467,23 +467,23 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
LOG.debug("Assigned based on * match");
ContainerProxy container = context.getMasterContext().getResourceAllocator().getContainer(
taskRequest.getContainerId());
- QueryUnitAttemptScheduleContext queryUnitContext = new QueryUnitAttemptScheduleContext(container.containerID,
+ TaskAttemptScheduleContext taskScheduleContext = new TaskAttemptScheduleContext(container.containerID,
container.getTaskHostName(), taskRequest.getCallback());
- QueryUnit task = SubQuery.newEmptyQueryUnit(context, queryUnitContext, subQuery, nextTaskId++);
+ Task task = SubQuery.newEmptyTask(context, taskScheduleContext, subQuery, nextTaskId++);
task.setFragment(scheduledFragments.getAllFragments());
subQuery.getEventHandler().handle(new TaskEvent(task.getId(), TaskEventType.T_SCHEDULE));
}
}
}
- private void assignTask(QueryUnitAttemptScheduleContext attemptContext, QueryUnitAttempt taskAttempt) {
- QueryUnitAttemptId attemptId = taskAttempt.getId();
- QueryUnitRequest taskAssign = new QueryUnitRequestImpl(
+ private void assignTask(TaskAttemptScheduleContext attemptContext, TaskAttempt taskAttempt) {
+ TaskAttemptId attemptId = taskAttempt.getId();
+ TaskRequest taskAssign = new TaskRequestImpl(
attemptId,
- new ArrayList<FragmentProto>(taskAttempt.getQueryUnit().getAllFragments()),
+ new ArrayList<FragmentProto>(taskAttempt.getTask().getAllFragments()),
"",
false,
- taskAttempt.getQueryUnit().getLogicalPlan().toJson(),
+ taskAttempt.getTask().getLogicalPlan().toJson(),
context.getMasterContext().getQueryContext(),
subQuery.getDataChannel(), subQuery.getBlock().getEnforcer());
if (checkIfInterQuery(subQuery.getMasterPlan(), subQuery.getBlock())) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java
index 64081f3..aced80c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java
@@ -21,8 +21,8 @@ package org.apache.tajo.master;
import com.google.protobuf.ByteString;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
-import org.apache.tajo.QueryUnitAttemptId;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskAttemptId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.conf.TajoConf;
@@ -84,7 +84,7 @@ public class NonForwardQueryResultScanner {
FragmentProto[] fragmentProtos = FragmentConvertor.toFragmentProtoArray(fragments.toArray(new Fragment[]{}));
this.taskContext = new TaskAttemptContext(
new QueryContext(tajoConf), null,
- new QueryUnitAttemptId(new QueryUnitId(new ExecutionBlockId(queryId, 1), 0), 0),
+ new TaskAttemptId(new TaskId(new ExecutionBlockId(queryId, 1), 0), 0),
fragmentProtos, null);
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
index 0d2acf7..4649d99 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.ContainerProtocol;
@@ -75,7 +75,7 @@ public class TajoContainerProxy extends ContainerProxy {
*
* @param taskAttemptId The TaskAttemptId to be killed.
*/
- public void killTaskAttempt(QueryUnitAttemptId taskAttemptId) {
+ public void killTaskAttempt(TaskAttemptId taskAttemptId) {
NettyClientBase tajoWorkerRpc = null;
try {
InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java b/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
index 4d10efe..183aeb5 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/ContainerAllocatorEventType.java
@@ -19,7 +19,7 @@
package org.apache.tajo.master.event;
public enum ContainerAllocatorEventType {
- // producer: QueryUnitAttempt, consumer: ContainerAllocator
+ // producer: TaskAttempt, consumer: ContainerAllocator
CONTAINER_REQ,
CONTAINER_DEALLOCATE,
CONTAINER_FAILED
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
index cab2202..5cf9887 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
@@ -19,24 +19,24 @@
package org.apache.tajo.master.event;
import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.master.container.TajoContainerId;
/**
* This event is sent to a running TaskAttempt on a worker.
*/
public class LocalTaskEvent extends AbstractEvent<LocalTaskEventType> {
- private final QueryUnitAttemptId taskAttemptId;
+ private final TaskAttemptId taskAttemptId;
private final TajoContainerId containerId;
- public LocalTaskEvent(QueryUnitAttemptId taskAttemptId, TajoContainerId containerId,
+ public LocalTaskEvent(TaskAttemptId taskAttemptId, TajoContainerId containerId,
LocalTaskEventType eventType) {
super(eventType);
this.taskAttemptId = taskAttemptId;
this.containerId = containerId;
}
- public QueryUnitAttemptId getTaskAttemptId() {
+ public TaskAttemptId getTaskAttemptId() {
return taskAttemptId;
}
[05/13] tajo git commit: TAJO-324: Rename the prefix 'QueryUnit' to
Task.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/TestTajoIds.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TestTajoIds.java b/tajo-core/src/test/java/org/apache/tajo/TestTajoIds.java
index 60b6c22..d15e282 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TestTajoIds.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TestTajoIds.java
@@ -64,10 +64,10 @@ public class TestTajoIds {
ExecutionBlockId subId = QueryIdFactory.newExecutionBlockId(queryId, 2);
assertEquals("eb_" + timeId +"_0001_000002", subId.toString());
- QueryUnitId qId = new QueryUnitId(subId, 5);
+ TaskId qId = new TaskId(subId, 5);
assertEquals("t_" + timeId + "_0001_000002_000005", qId.toString());
- QueryUnitAttemptId attemptId = new QueryUnitAttemptId(qId, 4);
+ TaskAttemptId attemptId = new TaskAttemptId(qId, 4);
assertEquals("ta_" + timeId + "_0001_000002_000005_04", attemptId.toString());
}
@@ -87,10 +87,10 @@ public class TestTajoIds {
ExecutionBlockId sid3 = QueryIdFactory.newExecutionBlockId(queryId1, 1);
assertEquals(sid1, sid3);
- QueryUnitId qid1 = new QueryUnitId(sid1, 9);
- QueryUnitId qid2 = new QueryUnitId(sid1, 10);
+ TaskId qid1 = new TaskId(sid1, 9);
+ TaskId qid2 = new TaskId(sid1, 10);
assertNotSame(qid1, qid2);
- QueryUnitId qid3 = new QueryUnitId(sid1, 9);
+ TaskId qid3 = new TaskId(sid1, 9);
assertEquals(qid1, qid3);
}
@@ -112,9 +112,9 @@ public class TestTajoIds {
assertEquals(1, sid2.compareTo(sid1));
assertEquals(0, sid3.compareTo(sid1));
- QueryUnitId qid1 = new QueryUnitId(sid1, 9);
- QueryUnitId qid2 = new QueryUnitId(sid1, 10);
- QueryUnitId qid3 = new QueryUnitId(sid1, 9);
+ TaskId qid1 = new TaskId(sid1, 9);
+ TaskId qid2 = new TaskId(sid1, 10);
+ TaskId qid3 = new TaskId(sid1, 9);
assertEquals(-1, qid1.compareTo(qid2));
assertEquals(1, qid2.compareTo(qid1));
assertEquals(0, qid3.compareTo(qid1));
@@ -131,12 +131,12 @@ public class TestTajoIds {
ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
assertEquals(sub1, sub2);
- QueryUnitId u1 = QueryIdFactory.newQueryUnitId(sub1);
- QueryUnitId u2 = new QueryUnitId(u1.getProto());
+ TaskId u1 = QueryIdFactory.newTaskId(sub1);
+ TaskId u2 = new TaskId(u1.getProto());
assertEquals(u1, u2);
- QueryUnitAttemptId attempt1 = new QueryUnitAttemptId(u1, 1);
- QueryUnitAttemptId attempt2 = new QueryUnitAttemptId(attempt1.getProto());
+ TaskAttemptId attempt1 = new TaskAttemptId(u1, 1);
+ TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto());
assertEquals(attempt1, attempt2);
}
@@ -151,12 +151,12 @@ public class TestTajoIds {
ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
assertEquals(sub1, sub2);
- QueryUnitId u1 = QueryIdFactory.newQueryUnitId(sub1);
- QueryUnitId u2 = new QueryUnitId(u1.getProto());
+ TaskId u1 = QueryIdFactory.newTaskId(sub1);
+ TaskId u2 = new TaskId(u1.getProto());
assertEquals(u1, u2);
- QueryUnitAttemptId attempt1 = new QueryUnitAttemptId(u1, 1);
- QueryUnitAttemptId attempt2 = new QueryUnitAttemptId(attempt1.getProto());
+ TaskAttemptId attempt1 = new TaskAttemptId(u1, 1);
+ TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto());
assertEquals(attempt1, attempt2);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 21dd7d5..d187071 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -787,17 +787,17 @@ public class TestTajoClient {
assertEquals(queryId.toString(), queryHistory.getQueryId());
assertEquals(2, queryHistory.getSubQueryHistoriesCount());
- List<SubQueryHistoryProto> queryUnitHistories =
+ List<SubQueryHistoryProto> taskHistories =
new ArrayList<SubQueryHistoryProto>(queryHistory.getSubQueryHistoriesList());
- Collections.sort(queryUnitHistories, new Comparator<SubQueryHistoryProto>() {
+ Collections.sort(taskHistories, new Comparator<SubQueryHistoryProto>() {
@Override
public int compare(SubQueryHistoryProto o1, SubQueryHistoryProto o2) {
return o1.getExecutionBlockId().compareTo(o2.getExecutionBlockId());
}
});
- assertEquals(5, queryUnitHistories.get(0).getTotalReadRows());
- assertEquals(1, queryUnitHistories.get(0).getTotalWriteRows());
- assertEquals(1, queryUnitHistories.get(1).getTotalReadRows());
- assertEquals(1, queryUnitHistories.get(1).getTotalWriteRows());
+ assertEquals(5, taskHistories.get(0).getTotalReadRows());
+ assertEquals(1, taskHistories.get(0).getTotalWriteRows());
+ assertEquals(1, taskHistories.get(1).getTotalReadRows());
+ assertEquals(1, taskHistories.get(1).getTotalWriteRows());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index 6a6aafb..8dfe1bb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -161,7 +161,7 @@ public class TestBNLJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -198,7 +198,7 @@ public class TestBNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLInnerJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(),
+ LocalTajoTestingUtility.newTaskAttemptId(),
merged, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index dc3c28d..c897461 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -166,7 +166,7 @@ public class TestBSTIndexExec {
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
Expr expr = analyzer.parse(QUERY);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
LogicalNode rootNode = optimizer.optimize(plan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index c0bf6ce..9f40062 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -126,7 +126,7 @@ public class TestExternalSortExec {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index ecd1c23..06f31ab 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -277,7 +277,7 @@ public class TestFullOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -316,7 +316,7 @@ public class TestFullOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -354,7 +354,7 @@ public class TestFullOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -393,7 +393,7 @@ public class TestFullOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec3");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged,
+ LocalTajoTestingUtility.newTaskAttemptId(), merged,
workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index a81979f..62a8034 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -326,7 +326,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -363,7 +363,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -400,7 +400,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -438,7 +438,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -479,7 +479,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin4");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -518,7 +518,7 @@ public class TestFullOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin5");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 4fe6ff2..64da88b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -160,7 +160,7 @@ public class TestHashAntiJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index 55e87d4..0738316 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -161,7 +161,7 @@ public class TestHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -204,7 +204,7 @@ public class TestHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
ctx.getQueryContext().setLong(SessionVars.HASH_JOIN_SIZE_LIMIT.keyname(), 100l);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index a2f1155..4e218c5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -165,7 +165,7 @@ public class TestHashSemiJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index 0477771..67ed358 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -281,7 +281,7 @@ public class TestLeftOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -312,7 +312,7 @@ public class TestLeftOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[1]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
@@ -353,7 +353,7 @@ public class TestLeftOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[2]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
@@ -395,7 +395,7 @@ public class TestLeftOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec3");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[3]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
@@ -437,7 +437,7 @@ public class TestLeftOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec4");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[4]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index 36dd77e..8043aa0 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -266,7 +266,7 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterNLJoinExec0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
@@ -308,7 +308,7 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[1]);
LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
@@ -352,7 +352,7 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[2]);
LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
@@ -397,7 +397,7 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec3");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[3]);
LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
@@ -441,7 +441,7 @@ public class TestLeftOuterNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec4");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[4]);
LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 10d4d33..ef3f401 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -174,7 +174,7 @@ public class TestMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index e976456..2be438f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -155,7 +155,7 @@ public class TestNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLCrossJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
@@ -184,7 +184,7 @@ public class TestNLJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLInnerJoin");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[1]);
LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index cce4ba7..6db76ae 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -250,7 +250,7 @@ public class TestPhysicalPlanner {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
@@ -281,7 +281,7 @@ public class TestPhysicalPlanner {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[16]);
@@ -310,7 +310,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[7]);
@@ -342,7 +342,7 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir(
"target/test-data/testHashGroupByPlanWithALLField");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[15]);
@@ -371,7 +371,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[]{frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[7]);
@@ -434,7 +434,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped1"));
@@ -485,7 +485,7 @@ public class TestPhysicalPlanner {
TaskAttemptContext ctx = new TaskAttemptContext(
queryContext,
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "maxOutput"));
@@ -531,7 +531,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped2"));
@@ -572,7 +572,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
ctx.setOutputPath(new Path(workDir, "grouped3"));
@@ -599,7 +599,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(enforcer);
ctx.setOutputPath(new Path(workDir, "grouped4"));
@@ -623,7 +623,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(enforcer);
ctx.setOutputPath(new Path(workDir, "grouped5"));
@@ -637,7 +637,7 @@ public class TestPhysicalPlanner {
public final void testPartitionedStorePlan() throws IOException, PlanningException {
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
- QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
+ TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan);
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] },
CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlan"));
ctx.setEnforcer(new Enforcer());
@@ -656,8 +656,8 @@ public class TestPhysicalPlanner {
TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());
FileSystem fs = sm.getFileSystem();
- QueryId queryId = id.getQueryUnitId().getExecutionBlockId().getQueryId();
- ExecutionBlockId ebId = id.getQueryUnitId().getExecutionBlockId();
+ QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId();
+ ExecutionBlockId ebId = id.getTaskId().getExecutionBlockId();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -706,7 +706,7 @@ public class TestPhysicalPlanner {
// Preparing working dir and input fragments
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(),
new Path(largeScore.getPath()), Integer.MAX_VALUE);
- QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
+ TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlanWithMaxFileSize");
// Setting session variables
@@ -770,7 +770,7 @@ public class TestPhysicalPlanner {
throws IOException, PlanningException {
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
- QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
+ TaskAttemptId id = LocalTajoTestingUtility.newTaskAttemptId(masterPlan);
Path workDir = CommonTestingUtil.getTestDir(
"target/test-data/testPartitionedStorePlanWithEmptyGroupingSet");
@@ -790,8 +790,8 @@ public class TestPhysicalPlanner {
TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());
FileSystem fs = sm.getFileSystem();
- QueryId queryId = id.getQueryUnitId().getExecutionBlockId().getQueryId();
- ExecutionBlockId ebId = id.getQueryUnitId().getExecutionBlockId();
+ QueryId queryId = id.getTaskId().getExecutionBlockId().getQueryId();
+ ExecutionBlockId ebId = id.getTaskId().getExecutionBlockId();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
@@ -839,7 +839,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[8]);
@@ -870,7 +870,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[9]);
@@ -898,7 +898,7 @@ public class TestPhysicalPlanner {
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[11]);
@@ -923,7 +923,7 @@ public class TestPhysicalPlanner {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
@@ -951,7 +951,7 @@ public class TestPhysicalPlanner {
public final void testEvalExpr() throws IOException, PlanningException {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEvalExpr");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] { }, workDir);
Expr expr = analyzer.parse(QUERIES[12]);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -988,7 +988,7 @@ public class TestPhysicalPlanner {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
Expr context = analyzer.parse(createIndexStmt[0]);
LogicalPlan plan = planner.createPlan(defaultContext, context);
@@ -1016,7 +1016,7 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(duplicateElimination[0]);
@@ -1050,7 +1050,7 @@ public class TestPhysicalPlanner {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(SORT_QUERY[0]);
@@ -1152,7 +1152,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT);
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -1174,7 +1174,7 @@ public class TestPhysicalPlanner {
enforcer = new Enforcer();
enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT);
ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -1202,7 +1202,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceHashAggregation(groupByNode.getPID());
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
@@ -1224,7 +1224,7 @@ public class TestPhysicalPlanner {
enforcer = new Enforcer();
enforcer.enforceSortAggregation(groupByNode.getPID(), null);
ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
+ LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
ctx.setEnforcer(enforcer);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index 3c78b12..a2b407b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -141,7 +141,7 @@ public class TestProgressExternalSortExec {
new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index 879ca21..e4ed595 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -241,7 +241,7 @@ public class TestRightOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
@@ -282,7 +282,7 @@ public class TestRightOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[1]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
@@ -323,7 +323,7 @@ public class TestRightOuterHashJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[2]);
LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 8bc00cc..112a341 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -325,7 +325,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -362,7 +362,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -398,7 +398,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -434,7 +434,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -472,7 +472,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin4");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
@@ -509,7 +509,7 @@ public class TestRightOuterMergeJoinExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin5");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir);
+ LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(enforcer);
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 8a61cab..2e093c1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -115,7 +115,7 @@ public class TestSortExec {
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility
- .newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
+ .newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
Expr context = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index cccbf2f..3dd1219 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -26,7 +26,7 @@ import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.master.querymaster.Query;
import org.apache.tajo.master.querymaster.QueryMasterTask;
-import org.apache.tajo.master.querymaster.QueryUnit;
+import org.apache.tajo.master.querymaster.Task;
import org.apache.tajo.master.querymaster.SubQuery;
import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
@@ -736,7 +736,7 @@ public class TestGroupByQuery extends QueryTestCaseBase {
assertTrue(!subQueries.isEmpty());
for (SubQuery subQuery: subQueries) {
if (subQuery.getId().toStringNoPrefix().endsWith("_000001")) {
- for (QueryUnit.IntermediateEntry eachInterm: subQuery.getHashShuffleIntermediateEntries()) {
+ for (Task.IntermediateEntry eachInterm: subQuery.getHashShuffleIntermediateEntries()) {
partitionIds.add(eachInterm.getPartId());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
index 3c96770..66d74c4 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java
@@ -25,10 +25,9 @@ import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.QueryId;
import org.apache.tajo.TestTajoIds;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.master.querymaster.QueryUnit;
-import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
+import org.apache.tajo.master.querymaster.Task;
+import org.apache.tajo.master.querymaster.Task.IntermediateEntry;
import org.apache.tajo.master.querymaster.Repartitioner;
-import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.FetchImpl;
@@ -61,7 +60,7 @@ public class TestRepartitioner {
}
for (int i = 0; i < 1000; i++) {
int partitionId = i % numPartition;
- IntermediateEntry entry = new IntermediateEntry(i, 0, partitionId, new QueryUnit.PullHost(hostName, port));
+ IntermediateEntry entry = new IntermediateEntry(i, 0, partitionId, new Task.PullHost(hostName, port));
entry.setEbId(sid);
entry.setVolume(10);
intermediateEntries.get(partitionId).add(entry);
@@ -71,7 +70,7 @@ public class TestRepartitioner {
new HashMap<Integer, Map<ExecutionBlockId, List<IntermediateEntry>>>();
for (Map.Entry<Integer, List<IntermediateEntry>> eachEntry: intermediateEntries.entrySet()) {
- FetchImpl fetch = new FetchImpl(new QueryUnit.PullHost(hostName, port), ShuffleType.HASH_SHUFFLE,
+ FetchImpl fetch = new FetchImpl(new Task.PullHost(hostName, port), ShuffleType.HASH_SHUFFLE,
sid, eachEntry.getKey(), eachEntry.getValue());
fetch.setName(sid.toString());
@@ -120,7 +119,7 @@ public class TestRepartitioner {
ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
FetchImpl [] fetches = new FetchImpl[12];
for (int i = 0; i < 12; i++) {
- fetches[i] = new FetchImpl(new QueryUnit.PullHost("localhost", 10000 + i), HASH_SHUFFLE, ebId, i / 2);
+ fetches[i] = new FetchImpl(new Task.PullHost("localhost", 10000 + i), HASH_SHUFFLE, ebId, i / 2);
}
int [] VOLUMES = {100, 80, 70, 30, 10, 5};
@@ -185,7 +184,7 @@ public class TestRepartitioner {
offset += pageLengths[j];
expectedTotalLength += pageLengths[j];
}
- IntermediateEntry interm = new IntermediateEntry(i, -1, -1, new QueryUnit.PullHost("" + i, i));
+ IntermediateEntry interm = new IntermediateEntry(i, -1, -1, new Task.PullHost("" + i, i));
interm.setPages(pages);
interm.setVolume(offset);
intermediateEntries.add(interm);
@@ -243,7 +242,7 @@ public class TestRepartitioner {
offset += pageLengths[j];
expectedTotalLength += pageLengths[j];
}
- IntermediateEntry interm = new IntermediateEntry(i, -1, 0, new QueryUnit.PullHost("" + i, i));
+ IntermediateEntry interm = new IntermediateEntry(i, -1, 0, new Task.PullHost("" + i, i));
interm.setPages(pages);
interm.setVolume(offset);
intermediateEntries.add(interm);
@@ -372,7 +371,7 @@ public class TestRepartitioner {
for (int j = 0; j < pageDatas.length; j++) {
pages.add(new Pair(pageDatas[j][0], (int) (pageDatas[j][1])));
}
- IntermediateEntry entry = new IntermediateEntry(-1, -1, 1, new QueryUnit.PullHost("host" + i , 9000));
+ IntermediateEntry entry = new IntermediateEntry(-1, -1, 1, new Task.PullHost("host" + i , 9000));
entry.setPages(pages);
entries.add(entry);
@@ -422,7 +421,7 @@ public class TestRepartitioner {
}
long expectedTotalLength = 0;
- QueryUnit.PullHost pullHost = new QueryUnit.PullHost("host", 0);
+ Task.PullHost pullHost = new Task.PullHost("host", 0);
for (int i = 0; i < 20; i++) {
List<Pair<Long, Integer>> pages = new ArrayList<Pair<Long, Integer>>();
@@ -481,7 +480,7 @@ public class TestRepartitioner {
@Test
public void testFetchImpl() {
ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
- QueryUnit.PullHost pullHost = new QueryUnit.PullHost("localhost", 0);
+ Task.PullHost pullHost = new Task.PullHost("localhost", 0);
FetchImpl expected = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1);
FetchImpl fetch2 = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestIntermediateEntry.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestIntermediateEntry.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestIntermediateEntry.java
index 114b232..7698987 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestIntermediateEntry.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestIntermediateEntry.java
@@ -29,7 +29,7 @@ import static org.junit.Assert.assertEquals;
public class TestIntermediateEntry {
@Test
public void testPage() {
- QueryUnit.IntermediateEntry interm = new QueryUnit.IntermediateEntry(-1, -1, 1, null);
+ Task.IntermediateEntry interm = new Task.IntermediateEntry(-1, -1, 1, null);
List<Pair<Long, Integer>> pages = new ArrayList<Pair<Long, Integer>>();
pages.add(new Pair(0L, 1441275));
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
deleted file mode 100644
index d64e4c7..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.querymaster;
-
-import org.apache.tajo.IntegrationTest;
-import org.apache.tajo.QueryTestCaseBase;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.worker.TajoWorker;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import java.sql.ResultSet;
-import java.util.*;
-
-import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
-import static org.junit.Assert.*;
-
-@Category(IntegrationTest.class)
-public class TestQueryUnitStatusUpdate extends QueryTestCaseBase {
-
- public TestQueryUnitStatusUpdate() {
- super(TajoConstants.DEFAULT_DATABASE_NAME);
- }
-
- @BeforeClass
- public static void setUp() throws Exception {
- conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
- }
-
- @Test
- public final void case1() throws Exception {
- // select l_linenumber, count(1) as unique_key from lineitem group by l_linenumber;
- ResultSet res = null;
- try {
- res = executeQuery();
-
- // tpch/lineitem.tbl
- long[] expectedNumRows = new long[]{5, 2, 2, 2};
- long[] expectedNumBytes = new long[]{604, 18, 18, 8};
- long[] expectedReadBytes = new long[]{604, 604, 18, 0};
-
- assertStatus(2, expectedNumRows, expectedNumBytes, expectedReadBytes);
- } finally {
- cleanupQuery(res);
- }
- }
-
- @Test
- public final void case2() throws Exception {
- // ExternalMergeSort
- ResultSet res = null;
- try {
- res = executeQuery();
-
- // tpch/lineitem.tbl
- long[] expectedNumRows = new long[]{5, 2, 2, 2, 2, 2};
- long[] expectedNumBytes = new long[]{604, 162, 162, 138, 138, 194};
- long[] expectedReadBytes = new long[]{604, 604, 162, 0, 138, 0};
-
- assertStatus(3, expectedNumRows, expectedNumBytes, expectedReadBytes);
- } finally {
- cleanupQuery(res);
- }
- }
-
-
- @Test
- public final void case3() throws Exception {
- // Partition Scan
- ResultSet res = null;
- try {
- createColumnPartitionedTable();
-
- /*
- |-eb_1404143727281_0002_000005
- |-eb_1404143727281_0002_000004 (order by)
- |-eb_1404143727281_0002_000003 (join)
- |-eb_1404143727281_0002_000002 (scan)
- |-eb_1404143727281_0002_000001 (scan, filter)
- */
- res = executeQuery();
-
- String actualResult = resultSetToString(res);
- System.out.println(actualResult);
-
- // in/out * subquery(4)
- long[] expectedNumRows = new long[]{2, 2, 5, 5, 7, 2, 2, 2};
- long[] expectedNumBytes = new long[]{8, 34, 20, 75, 109, 34, 34, 18};
- long[] expectedReadBytes = new long[]{8, 8, 20, 20, 109, 0, 34, 0};
-
- assertStatus(4, expectedNumRows, expectedNumBytes, expectedReadBytes);
- } finally {
- cleanupQuery(res);
- }
- }
-
- private void createColumnPartitionedTable() throws Exception {
- String tableName = CatalogUtil.normalizeIdentifier("ColumnPartitionedTable");
- ResultSet res = executeString(
- "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
- res.close();
-
- assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
- assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
- assertEquals(3,
- catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
-
- res = testBase.execute(
- "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_quantity from lineitem");
-
- res.close();
- }
-
- private void assertStatus(int numSubQueries,
- long[] expectedNumRows,
- long[] expectedNumBytes,
- long[] expectedReadBytes) throws Exception {
- List<TajoWorker> tajoWorkers = testingCluster.getTajoWorkers();
- Collection<QueryMasterTask> finishedTasks = null;
- for (TajoWorker eachWorker: tajoWorkers) {
- finishedTasks = eachWorker.getWorkerContext().getQueryMaster().getFinishedQueryMasterTasks();
- if (finishedTasks != null && !finishedTasks.isEmpty()) {
- break;
- }
- }
-
- assertNotNull(finishedTasks);
- assertTrue(!finishedTasks.isEmpty());
-
- List<QueryMasterTask> finishedTaskList = new ArrayList<QueryMasterTask>(finishedTasks);
-
- Collections.sort(finishedTaskList, new Comparator<QueryMasterTask>() {
- @Override
- public int compare(QueryMasterTask o1, QueryMasterTask o2) {
- return o2.getQueryId().compareTo(o1.getQueryId());
- }
- });
-
- Query query = finishedTaskList.get(0).getQuery();
-
- assertNotNull(query);
-
- List<SubQuery> subQueries = new ArrayList<SubQuery>(query.getSubQueries());
- assertEquals(numSubQueries, subQueries.size());
-
- Collections.sort(subQueries, new Comparator<SubQuery>() {
- @Override
- public int compare(SubQuery o1, SubQuery o2) {
- return o1.getId().compareTo(o2.getId());
- }
- });
-
- int index = 0;
- for (SubQuery eachSubQuery: subQueries) {
- TableStats inputStats = eachSubQuery.getInputStats();
- TableStats resultStats = eachSubQuery.getResultStats();
-
- assertNotNull(inputStats);
- assertEquals(expectedNumRows[index], inputStats.getNumRows().longValue());
- assertEquals(expectedNumBytes[index], inputStats.getNumBytes().longValue());
- assertEquals(expectedReadBytes[index], inputStats.getReadBytes().longValue());
-
- index++;
-
- assertNotNull(resultStats);
- assertEquals(expectedNumRows[index], resultStats.getNumRows().longValue());
- assertEquals(expectedNumBytes[index], resultStats.getNumBytes().longValue());
- assertEquals(expectedReadBytes[index], resultStats.getReadBytes().longValue());
-
- index++;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestTaskStatusUpdate.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestTaskStatusUpdate.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestTaskStatusUpdate.java
new file mode 100644
index 0000000..1c2a1a8
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestTaskStatusUpdate.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.querymaster;
+
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.worker.TajoWorker;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.sql.ResultSet;
+import java.util.*;
+
+import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
+import static org.junit.Assert.*;
+
+@Category(IntegrationTest.class)
+public class TestTaskStatusUpdate extends QueryTestCaseBase {
+
+ public TestTaskStatusUpdate() {
+ super(TajoConstants.DEFAULT_DATABASE_NAME);
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
+ }
+
+ @Test
+ public final void case1() throws Exception {
+ // select l_linenumber, count(1) as unique_key from lineitem group by l_linenumber;
+ ResultSet res = null;
+ try {
+ res = executeQuery();
+
+ // tpch/lineitem.tbl
+ long[] expectedNumRows = new long[]{5, 2, 2, 2};
+ long[] expectedNumBytes = new long[]{604, 18, 18, 8};
+ long[] expectedReadBytes = new long[]{604, 604, 18, 0};
+
+ assertStatus(2, expectedNumRows, expectedNumBytes, expectedReadBytes);
+ } finally {
+ cleanupQuery(res);
+ }
+ }
+
+ @Test
+ public final void case2() throws Exception {
+ // ExternalMergeSort
+ ResultSet res = null;
+ try {
+ res = executeQuery();
+
+ // tpch/lineitem.tbl
+ long[] expectedNumRows = new long[]{5, 2, 2, 2, 2, 2};
+ long[] expectedNumBytes = new long[]{604, 162, 162, 138, 138, 194};
+ long[] expectedReadBytes = new long[]{604, 604, 162, 0, 138, 0};
+
+ assertStatus(3, expectedNumRows, expectedNumBytes, expectedReadBytes);
+ } finally {
+ cleanupQuery(res);
+ }
+ }
+
+
+ @Test
+ public final void case3() throws Exception {
+ // Partition Scan
+ ResultSet res = null;
+ try {
+ createColumnPartitionedTable();
+
+ /*
+ |-eb_1404143727281_0002_000005
+ |-eb_1404143727281_0002_000004 (order by)
+ |-eb_1404143727281_0002_000003 (join)
+ |-eb_1404143727281_0002_000002 (scan)
+ |-eb_1404143727281_0002_000001 (scan, filter)
+ */
+ res = executeQuery();
+
+ String actualResult = resultSetToString(res);
+ System.out.println(actualResult);
+
+ // in/out * subquery(4)
+ long[] expectedNumRows = new long[]{2, 2, 5, 5, 7, 2, 2, 2};
+ long[] expectedNumBytes = new long[]{8, 34, 20, 75, 109, 34, 34, 18};
+ long[] expectedReadBytes = new long[]{8, 8, 20, 20, 109, 0, 34, 0};
+
+ assertStatus(4, expectedNumRows, expectedNumBytes, expectedReadBytes);
+ } finally {
+ cleanupQuery(res);
+ }
+ }
+
+ private void createColumnPartitionedTable() throws Exception {
+ String tableName = CatalogUtil.normalizeIdentifier("ColumnPartitionedTable");
+ ResultSet res = executeString(
+ "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ");
+ res.close();
+
+ assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
+ assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size());
+ assertEquals(3,
+ catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size());
+
+ res = testBase.execute(
+ "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_quantity from lineitem");
+
+ res.close();
+ }
+
+ private void assertStatus(int numSubQueries,
+ long[] expectedNumRows,
+ long[] expectedNumBytes,
+ long[] expectedReadBytes) throws Exception {
+ List<TajoWorker> tajoWorkers = testingCluster.getTajoWorkers();
+ Collection<QueryMasterTask> finishedTasks = null;
+ for (TajoWorker eachWorker: tajoWorkers) {
+ finishedTasks = eachWorker.getWorkerContext().getQueryMaster().getFinishedQueryMasterTasks();
+ if (finishedTasks != null && !finishedTasks.isEmpty()) {
+ break;
+ }
+ }
+
+ assertNotNull(finishedTasks);
+ assertTrue(!finishedTasks.isEmpty());
+
+ List<QueryMasterTask> finishedTaskList = new ArrayList<QueryMasterTask>(finishedTasks);
+
+ Collections.sort(finishedTaskList, new Comparator<QueryMasterTask>() {
+ @Override
+ public int compare(QueryMasterTask o1, QueryMasterTask o2) {
+ return o2.getQueryId().compareTo(o1.getQueryId());
+ }
+ });
+
+ Query query = finishedTaskList.get(0).getQuery();
+
+ assertNotNull(query);
+
+ List<SubQuery> subQueries = new ArrayList<SubQuery>(query.getSubQueries());
+ assertEquals(numSubQueries, subQueries.size());
+
+ Collections.sort(subQueries, new Comparator<SubQuery>() {
+ @Override
+ public int compare(SubQuery o1, SubQuery o2) {
+ return o1.getId().compareTo(o2.getId());
+ }
+ });
+
+ int index = 0;
+ for (SubQuery eachSubQuery: subQueries) {
+ TableStats inputStats = eachSubQuery.getInputStats();
+ TableStats resultStats = eachSubQuery.getResultStats();
+
+ assertNotNull(inputStats);
+ assertEquals(expectedNumRows[index], inputStats.getNumRows().longValue());
+ assertEquals(expectedNumBytes[index], inputStats.getNumBytes().longValue());
+ assertEquals(expectedReadBytes[index], inputStats.getReadBytes().longValue());
+
+ index++;
+
+ assertNotNull(resultStats);
+ assertEquals(expectedNumRows[index], resultStats.getNumRows().longValue());
+ assertEquals(expectedNumBytes[index], resultStats.getNumBytes().longValue());
+ assertEquals(expectedReadBytes[index], resultStats.getReadBytes().longValue());
+
+ index++;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/util/TestJSPUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/util/TestJSPUtil.java b/tajo-core/src/test/java/org/apache/tajo/util/TestJSPUtil.java
index 96aa5de..2e00138 100644
--- a/tajo-core/src/test/java/org/apache/tajo/util/TestJSPUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/util/TestJSPUtil.java
@@ -20,10 +20,10 @@ package org.apache.tajo.util;
import org.apache.hadoop.conf.Configuration;
import org.apache.tajo.ExecutionBlockId;
-import org.apache.tajo.QueryUnitId;
+import org.apache.tajo.TaskId;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent;
-import org.apache.tajo.master.querymaster.QueryUnit;
+import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent;
+import org.apache.tajo.master.querymaster.Task;
import org.junit.Test;
import java.util.ArrayList;
@@ -34,52 +34,52 @@ import static org.junit.Assert.assertEquals;
public class TestJSPUtil {
@Test
- public void testSortQueryUnit() throws Exception {
- List<QueryUnit> queryUnits = new ArrayList<QueryUnit>();
+ public void testSortTask() throws Exception {
+ List<Task> tasks = new ArrayList<Task>();
Configuration conf = new TajoConf();
- QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext scheduleContext =
- new QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext();
+ TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext scheduleContext =
+ new TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext();
ExecutionBlockId ebId = TajoIdUtils.createExecutionBlockId("eb_000001_00001_00001");
for (int i = 0; i < 10; i++) {
- QueryUnitId id = new QueryUnitId(ebId, i);
- QueryUnit queryUnit = new QueryUnit(conf, scheduleContext, id, true, null);
- queryUnits.add(queryUnit);
+ TaskId id = new TaskId(ebId, i);
+ Task task = new Task(conf, scheduleContext, id, true, null);
+ tasks.add(task);
int launchTime = i + 1;
int runningTime = i + 1;
if(i < 9) {
- queryUnit.setLaunchTime(launchTime);
- queryUnit.setFinishTime(launchTime + runningTime);
+ task.setLaunchTime(launchTime);
+ task.setFinishTime(launchTime + runningTime);
}
}
- Collections.shuffle(queryUnits);
+ Collections.shuffle(tasks);
- QueryUnit[] queryUnitArray = queryUnits.toArray(new QueryUnit[]{});
- JSPUtil.sortQueryUnitArray(queryUnitArray, "id", "asc");
+ Task[] taskArray = tasks.toArray(new Task[]{});
+ JSPUtil.sortTaskArray(taskArray, "id", "asc");
for (int i = 0; i < 10; i++) {
- assertEquals(i, queryUnitArray[i].getId().getId());
+ assertEquals(i, taskArray[i].getId().getId());
}
- queryUnitArray = queryUnits.toArray(new QueryUnit[]{});
- JSPUtil.sortQueryUnitArray(queryUnitArray, "id", "desc");
+ taskArray = tasks.toArray(new Task[]{});
+ JSPUtil.sortTaskArray(taskArray, "id", "desc");
for (int i = 0; i < 10; i++) {
- assertEquals(9 - i, queryUnitArray[i].getId().getId());
+ assertEquals(9 - i, taskArray[i].getId().getId());
}
- queryUnitArray = queryUnits.toArray(new QueryUnit[]{});
- JSPUtil.sortQueryUnitArray(queryUnitArray, "runTime", "asc");
- assertEquals(0, queryUnitArray[0].getId().getId());
- assertEquals(9, queryUnitArray[9].getId().getId());
+ taskArray = tasks.toArray(new Task[]{});
+ JSPUtil.sortTaskArray(taskArray, "runTime", "asc");
+ assertEquals(0, taskArray[0].getId().getId());
+ assertEquals(9, taskArray[9].getId().getId());
- queryUnitArray = queryUnits.toArray(new QueryUnit[]{});
- JSPUtil.sortQueryUnitArray(queryUnitArray, "runTime", "desc");
- assertEquals(8, queryUnitArray[0].getId().getId());
- assertEquals(9, queryUnitArray[9].getId().getId());
+ taskArray = tasks.toArray(new Task[]{});
+ JSPUtil.sortTaskArray(taskArray, "runTime", "desc");
+ assertEquals(8, taskArray[0].getId().getId());
+ assertEquals(9, taskArray[9].getId().getId());
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java b/tajo-core/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java
index d5b737f..8d6a94d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java
+++ b/tajo-core/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java
@@ -30,7 +30,6 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.master.querymaster.QueryInfo;
import org.apache.tajo.util.TajoIdUtils;
-import org.apache.tajo.worker.TaskHistory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -134,13 +133,13 @@ public class TestHistoryWriterReader extends QueryTestCaseBase {
subQueryHistory.setExecutionBlockId(ebId.toString());
subQueryHistory.setStartTime(startTime + i);
- List<QueryUnitHistory> queryUnitHistories = new ArrayList<QueryUnitHistory>();
+ List<TaskHistory> taskHistories = new ArrayList<TaskHistory>();
for (int j = 0; j < 5; j++) {
- QueryUnitHistory queryUnitHistory = new QueryUnitHistory();
- queryUnitHistory.setId(QueryIdFactory.newQueryUnitAttemptId(QueryIdFactory.newQueryUnitId(ebId), 1).toString());
- queryUnitHistories.add(queryUnitHistory);
+ TaskHistory taskHistory = new TaskHistory();
+ taskHistory.setId(QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(ebId), 1).toString());
+ taskHistories.add(taskHistory);
}
- subQueryHistory.setQueryUnits(queryUnitHistories);
+ subQueryHistory.setTasks(taskHistories);
subQueries.add(subQueryHistory);
}
queryHistory.setSubQueryHistories(subQueries);
@@ -175,16 +174,16 @@ public class TestHistoryWriterReader extends QueryTestCaseBase {
assertEquals(ebId, subQueryHistory.getExecutionBlockId());
assertEquals(startTime + i, subQueryHistory.getStartTime());
- // QueryUnitHistory is stored in the other file.
- assertNull(subQueryHistory.getQueryUnits());
+ // TaskHistory is stored in the other file.
+ assertNull(subQueryHistory.getTasks());
- List<QueryUnitHistory> queryUnits = reader.getQueryUnitHistory(queryId.toString(), ebId);
- assertNotNull(queryUnits);
- assertEquals(5, queryUnits.size());
+ List<TaskHistory> tasks = reader.getTaskHistory(queryId.toString(), ebId);
+ assertNotNull(tasks);
+ assertEquals(5, tasks.size());
for (int j = 0; j < 5; j++) {
- QueryUnitHistory queryUnitHistory = queryUnits.get(j);
- assertEquals(subQueries.get(i).getQueryUnits().get(j).getId(), queryUnitHistory.getId());
+ TaskHistory taskHistory = tasks.get(j);
+ assertEquals(subQueries.get(i).getTasks().get(j).getId(), taskHistory.getId());
}
}
} finally {
@@ -210,13 +209,13 @@ public class TestHistoryWriterReader extends QueryTestCaseBase {
.setNumBytes(100)
.build();
long startTime = System.currentTimeMillis() - 2000;
- QueryUnitAttemptId id1 = TajoIdUtils.parseQueryUnitAttemptId("ta_1412326813565_0001_000001_000001_00");
- TaskHistory taskHistory1 = new TaskHistory(
+ TaskAttemptId id1 = TajoIdUtils.parseTaskAttemptId("ta_1412326813565_0001_000001_000001_00");
+ org.apache.tajo.worker.TaskHistory taskHistory1 = new org.apache.tajo.worker.TaskHistory(
id1, TaskAttemptState.TA_SUCCEEDED, 1.0f, startTime, System.currentTimeMillis(), tableStats);
writer.appendHistory(taskHistory1);
- QueryUnitAttemptId id2 = TajoIdUtils.parseQueryUnitAttemptId("ta_1412326813565_0001_000001_000002_00");
- TaskHistory taskHistory2 = new TaskHistory(
+ TaskAttemptId id2 = TajoIdUtils.parseTaskAttemptId("ta_1412326813565_0001_000001_000002_00");
+ org.apache.tajo.worker.TaskHistory taskHistory2 = new org.apache.tajo.worker.TaskHistory(
id2, TaskAttemptState.TA_SUCCEEDED, 1.0f, startTime, System.currentTimeMillis() - 500, tableStats);
writer.appendHistory(taskHistory2);
@@ -232,14 +231,14 @@ public class TestHistoryWriterReader extends QueryTestCaseBase {
assertTrue(fs.exists(taskParentPath));
HistoryReader reader = new HistoryReader("127.0.0.1:28090", tajoConf);
- TaskHistory foundTaskHistory = reader.getTaskHistory(id1.toString(), startTime);
+ org.apache.tajo.worker.TaskHistory foundTaskHistory = reader.getTaskHistory(id1.toString(), startTime);
assertNotNull(foundTaskHistory);
- assertEquals(id1, foundTaskHistory.getQueryUnitAttemptId());
+ assertEquals(id1, foundTaskHistory.getTaskAttemptId());
assertEquals(taskHistory1, foundTaskHistory);
foundTaskHistory = reader.getTaskHistory(id2.toString(), startTime);
assertNotNull(foundTaskHistory);
- assertEquals(id2, foundTaskHistory.getQueryUnitAttemptId());
+ assertEquals(id2, foundTaskHistory.getTaskAttemptId());
assertEquals(taskHistory2, foundTaskHistory);
foundTaskHistory = reader.getTaskHistory("ta_1412326813565_0001_000001_000003_00", startTime);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
index fa90b61..d320077 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestHistory.java
@@ -20,7 +20,7 @@ package org.apache.tajo.worker;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.service.Service;
-import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.TpchTestBase;
@@ -107,13 +107,13 @@ public class TestHistory {
assertTrue(history.size() > 0);
assertEquals(Service.STATE.STOPPED, history.getState());
- Map.Entry<QueryUnitAttemptId, TaskHistory> entry =
+ Map.Entry<TaskAttemptId, TaskHistory> entry =
history.getTaskHistoryMap().entrySet().iterator().next();
- QueryUnitAttemptId queryUnitAttemptId = entry.getKey();
+ TaskAttemptId taskAttemptId = entry.getKey();
TaskHistory taskHistory = entry.getValue();
assertEquals(TajoProtos.TaskAttemptState.TA_SUCCEEDED, taskHistory.getState());
- assertEquals(queryUnitAttemptId, taskHistory.getQueryUnitAttemptId());
+ assertEquals(taskAttemptId, taskHistory.getTaskAttemptId());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5c852b79/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index 5a93538..200ba31 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -148,7 +148,7 @@ public class TestRangeRetrieverHandler {
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE);
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(),
+ LocalTajoTestingUtility.newTaskAttemptId(),
new FileFragment[] {frags[0]}, testDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(SORT_QUERY[0]);
@@ -272,7 +272,7 @@ public class TestRangeRetrieverHandler {
TaskAttemptContext
ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newQueryUnitAttemptId(),
+ LocalTajoTestingUtility.newTaskAttemptId(),
new FileFragment[] {frags[0]}, testDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(SORT_QUERY[1]);
[12/13] tajo git commit: TAJO-1259: A title in catalog configuration
document is different from others. (Jongyoung Park via hyunsik)
Posted by ji...@apache.org.
TAJO-1259: A title in catalog configuration document is different from others. (Jongyoung Park via hyunsik)
Closes #310
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/34131074
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/34131074
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/34131074
Branch: refs/heads/index_support
Commit: 3413107469a3ce9eb9cf46b1ad3b81bbd4f97d9a
Parents: b7c8cf7
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Dec 19 13:00:20 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Dec 19 13:00:20 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../src/main/sphinx/configuration/catalog_configuration.rst | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/34131074/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 54a4ef4..e30e24c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -115,6 +115,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1259: A title in catalog configuration document is different from others.
+ (Jongyoung Park via hyunsik)
+
TAJO-1232: Implicit groupby queries with LIMIT lead to wrong results.
(jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/34131074/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst b/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst
index 6d36d3f..acf05f9 100644
--- a/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst
+++ b/tajo-docs/src/main/sphinx/configuration/catalog_configuration.rst
@@ -126,9 +126,9 @@ All configurations for using MariaDBStore is compatible with MySQLStore except f
</property>
-----------------------------------
+==================================
HCatalogStore Configuration
-----------------------------------
+==================================
Tajo support HCatalogStore to integrate with hive. If you want to use HCatalogStore, you just do as follows.