You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/01/28 04:55:24 UTC
[1/4] tajo git commit: TAJO-1318: Unit test failure after miniDFS
cluster restart. (jinho)
Repository: tajo
Updated Branches:
refs/heads/index_support 9d48a40ad -> 7cafc33b0
TAJO-1318: Unit test failure after miniDFS cluster restart. (jinho)
Closes #360
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5ba8e383
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5ba8e383
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5ba8e383
Branch: refs/heads/index_support
Commit: 5ba8e383c61e6074e84dbd07a3b150370ccbaea1
Parents: a15b5fa
Author: jhkim <jh...@apache.org>
Authored: Tue Jan 27 16:00:19 2015 +0900
Committer: jhkim <jh...@apache.org>
Committed: Tue Jan 27 16:00:19 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../java/org/apache/tajo/worker/TajoWorker.java | 7 ++++++
.../org/apache/tajo/TajoTestingCluster.java | 24 ++++++++------------
.../org/apache/tajo/storage/StorageManager.java | 3 ++-
4 files changed, 21 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5ba8e383/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6969bf1..eb1c3dc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -171,6 +171,8 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1318: Unit test failure after miniDFS cluster restart. (jinho)
+
TAJO-1289: History reader fails to get the query information after
a successful query execution. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5ba8e383/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
index 4003014..c217e3e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java
@@ -49,6 +49,7 @@ import org.apache.tajo.rule.EvaluationFailedException;
import org.apache.tajo.rule.SelfDiagnosisRuleEngine;
import org.apache.tajo.rule.SelfDiagnosisRuleSession;
import org.apache.tajo.storage.HashShuffleAppenderManager;
+import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.JvmPauseMonitor;
import org.apache.tajo.util.NetUtils;
@@ -395,6 +396,12 @@ public class TajoWorker extends CompositeService {
}
}
+ try {
+ StorageManager.close();
+ } catch (IOException ie) {
+ LOG.error(ie.getMessage(), ie);
+ }
+
if(workerSystemMetrics != null) {
workerSystemMetrics.stop();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5ba8e383/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index e548b81..64b38ac 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -45,11 +45,11 @@ import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider;
import org.apache.tajo.master.QueryInProgress;
import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.master.rm.TajoWorkerResourceManager;
+import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider;
import org.apache.tajo.querymaster.Query;
import org.apache.tajo.querymaster.Stage;
import org.apache.tajo.querymaster.StageState;
-import org.apache.tajo.master.rm.TajoWorkerResourceManager;
-import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.NetUtils;
@@ -59,7 +59,10 @@ import java.io.*;
import java.net.InetSocketAddress;
import java.net.URL;
import java.sql.ResultSet;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.UUID;
public class TajoTestingCluster {
private static Log LOG = LogFactory.getLog(TajoTestingCluster.class);
@@ -237,18 +240,14 @@ public class TajoTestingCluster {
* @throws java.io.IOException
*/
public MiniDFSCluster startMiniDFSCluster(int servers,
- final File dir,
+ File dir,
final String hosts[])
throws IOException {
if (dir == null) {
- this.clusterTestBuildDir = setupClusterTestBuildDir();
- } else {
- this.clusterTestBuildDir = dir;
+ dir = setupClusterTestBuildDir();
}
- System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA,
- this.clusterTestBuildDir.toString());
-
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString());
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf));
@@ -514,10 +513,7 @@ public class TajoTestingCluster {
this.clusterTestBuildDir = testBuildPath == null?
setupClusterTestBuildDir() : new File(testBuildPath);
- System.setProperty(TEST_DIRECTORY_KEY,
- this.clusterTestBuildDir.getAbsolutePath());
-
- startMiniDFSCluster(numDataNodes, this.clusterTestBuildDir, dataNodeHosts);
+ startMiniDFSCluster(numDataNodes, setupClusterTestBuildDir(), dataNodeHosts);
this.dfsCluster.waitClusterUp();
hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5ba8e383/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index 3b56249..34caa80 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -242,12 +242,13 @@ public abstract class StorageManager {
* Close StorageManager
* @throws java.io.IOException
*/
- public void close() throws IOException {
+ public static void close() throws IOException {
synchronized(storageManagers) {
for (StorageManager eachStorageManager: storageManagers.values()) {
eachStorageManager.closeStorageManager();
}
}
+ clearCache();
}
/**
[2/4] tajo git commit: TAJO-1312: Stage causes Invalid event error:
SQ_SHUFFLE_REPORT at KILLED. (jinho)
Posted by ji...@apache.org.
TAJO-1312: Stage causes Invalid event error: SQ_SHUFFLE_REPORT at KILLED. (jinho)
Closes #359
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/d7ee6cd6
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/d7ee6cd6
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/d7ee6cd6
Branch: refs/heads/index_support
Commit: d7ee6cd682769f4c56aa1c053dc0bb4071813202
Parents: 5ba8e38
Author: jhkim <jh...@apache.org>
Authored: Tue Jan 27 16:26:22 2015 +0900
Committer: jhkim <jh...@apache.org>
Committed: Tue Jan 27 16:26:22 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../org/apache/tajo/master/QueryInProgress.java | 1 +
.../java/org/apache/tajo/querymaster/Stage.java | 2 +
.../java/org/apache/tajo/querymaster/Task.java | 5 +-
.../tajo/worker/TajoWorkerClientService.java | 4 +-
.../org/apache/tajo/TajoTestingCluster.java | 28 ++++++---
.../master/scheduler/TestFifoScheduler.java | 8 +--
.../apache/tajo/querymaster/TestKillQuery.java | 60 +++++++++++++++++++-
8 files changed, 92 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index eb1c3dc..ceacf97 100644
--- a/CHANGES
+++ b/CHANGES
@@ -171,6 +171,9 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1312: Stage causes Invalid event error: SQ_SHUFFLE_REPORT
+ at KILLED. (jinho)
+
TAJO-1318: Unit test failure after miniDFS cluster restart. (jinho)
TAJO-1289: History reader fails to get the query information after
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
index 7e2c05f..e7371dd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
@@ -168,6 +168,7 @@ public class QueryInProgress {
queryMasterRpcClient.executeQuery(null, builder.build(), NullCallback.get());
querySubmitted.set(true);
+ getQueryInfo().setQueryState(TajoProtos.QueryState.QUERY_MASTER_LAUNCHED);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 13394f8..208d4a6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -257,6 +257,8 @@ public class Stage implements EventHandler<StageEvent> {
StageEventType.SQ_START,
StageEventType.SQ_KILL,
StageEventType.SQ_CONTAINER_ALLOCATED,
+ StageEventType.SQ_SHUFFLE_REPORT,
+ StageEventType.SQ_STAGE_COMPLETED,
StageEventType.SQ_FAILED))
// Transitions from FAILED state
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
index 1c6a9a3..ad01b62 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java
@@ -172,7 +172,10 @@ public class Task implements EventHandler<TaskEvent> {
// Ignore-able transitions
.addTransition(TaskState.KILLED, TaskState.KILLED,
EnumSet.of(
- TaskEventType.T_KILL, TaskEventType.T_ATTEMPT_SUCCEEDED, TaskEventType.T_ATTEMPT_FAILED))
+ TaskEventType.T_KILL,
+ TaskEventType.T_SCHEDULE,
+ TaskEventType.T_ATTEMPT_SUCCEEDED,
+ TaskEventType.T_ATTEMPT_FAILED))
.installTopology();
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
index 2ae4bed..0b815d8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
@@ -118,8 +118,8 @@ public class TajoWorkerClientService extends AbstractService {
try {
QueryId queryId = new QueryId(request.getQueryId());
- QueryMasterTask queryMasterTask = workerContext.getQueryMaster().getQueryMasterTask(queryId);
- QueryHistory queryHistory = null;
+ QueryMasterTask queryMasterTask = workerContext.getQueryMaster().getQueryMasterTask(queryId, true);
+ QueryHistory queryHistory;
if (queryMasterTask == null) {
queryHistory = workerContext.getHistoryReader().getQueryHistory(queryId.toString());
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 64b38ac..8714fc4 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -43,11 +43,11 @@ import org.apache.tajo.client.TajoClientUtil;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider;
-import org.apache.tajo.master.QueryInProgress;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.rm.TajoWorkerResourceManager;
import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider;
import org.apache.tajo.querymaster.Query;
+import org.apache.tajo.querymaster.QueryMasterTask;
import org.apache.tajo.querymaster.Stage;
import org.apache.tajo.querymaster.StageState;
import org.apache.tajo.util.CommonTestingUtil;
@@ -772,21 +772,20 @@ public class TajoTestingCluster {
}
}
- public void waitForQueryRunning(QueryId queryId) throws Exception {
- waitForQueryRunning(queryId, 50);
+ public void waitForQuerySubmitted(QueryId queryId) throws Exception {
+ waitForQuerySubmitted(queryId, 50);
}
- public void waitForQueryRunning(QueryId queryId, int delay) throws Exception {
- QueryInProgress qip = null;
+ public void waitForQuerySubmitted(QueryId queryId, int delay) throws Exception {
+ QueryMasterTask qmt = null;
int i = 0;
- while (qip == null || TajoClientUtil.isQueryWaitingForSchedule(qip.getQueryInfo().getQueryState())) {
+ while (qmt == null || TajoClientUtil.isQueryWaitingForSchedule(qmt.getState())) {
try {
Thread.sleep(delay);
- if(qip == null){
- TajoMaster master = getMaster();
- qip = master.getContext().getQueryJobManager().getQueryInProgress(queryId);
+ if (qmt == null) {
+ qmt = getQueryMasterTask(queryId);
}
} catch (InterruptedException e) {
}
@@ -822,4 +821,15 @@ public class TajoTestingCluster {
}
}
}
+
+ public QueryMasterTask getQueryMasterTask(QueryId queryId) {
+ QueryMasterTask qmt = null;
+ for (TajoWorker worker : getTajoWorkers()) {
+ qmt = worker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId, true);
+ if (qmt != null && queryId.equals(qmt.getQueryId())) {
+ break;
+ }
+ }
+ return qmt;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java b/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
index e0c30a8..0a8a51c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/scheduler/TestFifoScheduler.java
@@ -68,7 +68,7 @@ public class TestFifoScheduler {
QueryId queryId = new QueryId(res.getQueryId());
QueryId queryId2 = new QueryId(res2.getQueryId());
- cluster.waitForQueryRunning(queryId);
+ cluster.waitForQuerySubmitted(queryId);
client.killQuery(queryId2);
assertEquals(TajoProtos.QueryState.QUERY_KILLED, client.getQueryStatus(queryId2).getState());
}
@@ -82,7 +82,7 @@ public class TestFifoScheduler {
QueryId queryId = new QueryId(res.getQueryId());
QueryId queryId2 = new QueryId(res2.getQueryId());
- cluster.waitForQueryRunning(queryId);
+ cluster.waitForQuerySubmitted(queryId);
assertEquals(TajoProtos.QueryState.QUERY_SUCCEEDED, client.getQueryStatus(queryId2).getState());
ResultSet resSet = TajoClientUtil.createResultSet(conf, client, res2);
@@ -101,9 +101,9 @@ public class TestFifoScheduler {
QueryId queryId3 = new QueryId(res3.getQueryId());
QueryId queryId4 = new QueryId(res4.getQueryId());
- cluster.waitForQueryRunning(queryId);
+ cluster.waitForQuerySubmitted(queryId);
- assertTrue(TajoClientUtil.isQueryRunning(client.getQueryStatus(queryId).getState()));
+ assertFalse(TajoClientUtil.isQueryComplete(client.getQueryStatus(queryId).getState()));
assertEquals(TajoProtos.QueryState.QUERY_MASTER_INIT, client.getQueryStatus(queryId2).getState());
assertEquals(TajoProtos.QueryState.QUERY_MASTER_INIT, client.getQueryStatus(queryId3).getState());
http://git-wip-us.apache.org/repos/asf/tajo/blob/d7ee6cd6/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
index bd899cd..42ad8da 100644
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
@@ -18,6 +18,7 @@
package org.apache.tajo.querymaster;
+import com.google.common.collect.Lists;
import org.apache.tajo.*;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.benchmark.TPCH;
@@ -29,18 +30,22 @@ import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.planner.global.GlobalPlanner;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.master.event.QueryEvent;
import org.apache.tajo.master.event.QueryEventType;
-import org.apache.tajo.session.Session;
+import org.apache.tajo.master.event.StageEvent;
+import org.apache.tajo.master.event.StageEventType;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.session.Session;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.util.List;
import static org.junit.Assert.*;
@@ -48,6 +53,9 @@ public class TestKillQuery {
private static TajoTestingCluster cluster;
private static TajoConf conf;
private static TajoClient client;
+ private static String queryStr = "select t1.l_orderkey, t1.l_partkey, t2.c_custkey " +
+ "from lineitem t1 join customer t2 " +
+ "on t1.l_orderkey = t2.c_custkey order by t1.l_orderkey";
@BeforeClass
public static void setUp() throws Exception {
@@ -59,6 +67,11 @@ public class TestKillQuery {
client.executeQueryAndGetResult("create external table default.lineitem (l_orderkey int, l_partkey int) "
+ "using text location 'file://" + file.getAbsolutePath() + "'");
assertTrue(client.existTable("default.lineitem"));
+
+ file = TPCH.getDataFile("customer");
+ client.executeQueryAndGetResult("create external table default.customer (c_custkey int, c_name text) "
+ + "using text location 'file://" + file.getAbsolutePath() + "'");
+ assertTrue(client.existTable("default.customer"));
}
@AfterClass
@@ -73,11 +86,10 @@ public class TestKillQuery {
QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
Session session = LocalTajoTestingUtility.createDummySession();
CatalogService catalog = cluster.getMaster().getCatalog();
- String query = "select l_orderkey, l_partkey from lineitem group by l_orderkey, l_partkey order by l_orderkey";
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(conf);
- Expr expr = analyzer.parse(query);
+ Expr expr = analyzer.parse(queryStr);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
@@ -122,4 +134,46 @@ public class TestKillQuery {
}
queryMasterTask.stop();
}
+
+ @Test
+ public final void testIgnoreStageStateFromKilled() throws Exception {
+
+ ClientProtos.SubmitQueryResponse res = client.executeQuery(queryStr);
+ QueryId queryId = new QueryId(res.getQueryId());
+ cluster.waitForQuerySubmitted(queryId);
+
+ QueryMasterTask qmt = cluster.getQueryMasterTask(queryId);
+ Query query = qmt.getQuery();
+
+ query.handle(new QueryEvent(queryId, QueryEventType.KILL));
+
+ try{
+ cluster.waitForQueryState(query, TajoProtos.QueryState.QUERY_KILLED, 50);
+ } finally {
+ assertEquals(TajoProtos.QueryState.QUERY_KILLED, query.getSynchronizedState());
+ }
+
+ List<Stage> stages = Lists.newArrayList(query.getStages());
+ Stage lastStage = stages.get(stages.size() - 1);
+
+ assertEquals(StageState.KILLED, lastStage.getSynchronizedState());
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_START,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_START));
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_KILL,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_KILL));
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_CONTAINER_ALLOCATED,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_CONTAINER_ALLOCATED));
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_SHUFFLE_REPORT,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_SHUFFLE_REPORT));
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_STAGE_COMPLETED,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_STAGE_COMPLETED));
+
+ lastStage.getStateMachine().doTransition(StageEventType.SQ_FAILED,
+ new StageEvent(lastStage.getId(), StageEventType.SQ_FAILED));
+ }
}
[4/4] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7cafc33b
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7cafc33b
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7cafc33b
Branch: refs/heads/index_support
Commit: 7cafc33b0c564e0d10f80cf3f25be2262ef1880f
Parents: 9d48a40 c429c97
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Jan 28 12:55:14 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Jan 28 12:55:14 2015 +0900
----------------------------------------------------------------------
CHANGES | 7 +
.../java/org/apache/tajo/conf/TajoConf.java | 2 +-
.../org/apache/tajo/master/QueryInProgress.java | 1 +
.../java/org/apache/tajo/querymaster/Stage.java | 2 +
.../java/org/apache/tajo/querymaster/Task.java | 5 +-
.../java/org/apache/tajo/worker/TajoWorker.java | 7 +
.../tajo/worker/TajoWorkerClientService.java | 4 +-
.../org/apache/tajo/TajoTestingCluster.java | 52 +++----
.../tajo/engine/query/TestHBaseTable.java | 135 +------------------
.../master/scheduler/TestFifoScheduler.java | 8 +-
.../apache/tajo/querymaster/TestKillQuery.java | 60 ++++++++-
tajo-dist/src/main/bin/tajo | 54 ++++----
.../org/apache/tajo/storage/StorageManager.java | 3 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 13 +-
14 files changed, 152 insertions(+), 201 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7cafc33b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7cafc33b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7cafc33b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
index b5c745a,42ad8da..afcc4d9
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
@@@ -73,11 -86,10 +86,10 @@@ public class TestKillQuery
QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
Session session = LocalTajoTestingUtility.createDummySession();
CatalogService catalog = cluster.getMaster().getCatalog();
- String query = "select l_orderkey, l_partkey from lineitem group by l_orderkey, l_partkey order by l_orderkey";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
- Expr expr = analyzer.parse(query);
+ Expr expr = analyzer.parse(queryStr);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
optimizer.optimize(plan);
[3/4] tajo git commit: TAJO-1319: Tajo can't find HBase configuration
file. (jaehwa)
Posted by ji...@apache.org.
TAJO-1319: Tajo can't find HBase configuration file. (jaehwa)
Closes #361
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/c429c971
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/c429c971
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/c429c971
Branch: refs/heads/index_support
Commit: c429c9710ff0c5791a17419d29a5fc7fdea60e4d
Parents: d7ee6cd
Author: JaeHwa Jung <bl...@apache.org>
Authored: Tue Jan 27 22:24:08 2015 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Tue Jan 27 22:24:08 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../java/org/apache/tajo/conf/TajoConf.java | 2 +-
.../tajo/engine/query/TestHBaseTable.java | 135 +------------------
tajo-dist/src/main/bin/tajo | 54 ++++----
.../tajo/storage/hbase/HBaseStorageManager.java | 13 +-
5 files changed, 39 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/c429c971/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index ceacf97..453e163 100644
--- a/CHANGES
+++ b/CHANGES
@@ -171,6 +171,8 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1319: Tajo can't find HBase configuration file. (jaehwa)
+
TAJO-1312: Stage causes Invalid event error: SQ_SHUFFLE_REPORT
at KILLED. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/c429c971/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index 195743a..1bb96bc 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -53,7 +53,7 @@ public class TajoConf extends Configuration {
Configuration.addDefaultResource("storage-site.xml");
Configuration.addDefaultResource("tajo-default.xml");
Configuration.addDefaultResource("tajo-site.xml");
-
+
for (ConfVars confVars: ConfVars.values()) {
vars.put(confVars.keyname(), confVars);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c429c971/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
index 741a807..7ca766f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java
@@ -61,10 +61,16 @@ import static org.junit.Assert.assertEquals;
public class TestHBaseTable extends QueryTestCaseBase {
private static final Log LOG = LogFactory.getLog(TestHBaseTable.class);
+ private static String hostName,zkPort;
+
@BeforeClass
public static void beforeClass() {
try {
testingCluster.getHBaseUtil().startHBaseCluster();
+ hostName = InetAddress.getLocalHost().getHostName();
+ zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ assertNotNull(hostName);
+ assertNotNull(zkPort);
} catch (Exception e) {
e.printStackTrace();
}
@@ -99,44 +105,10 @@ public class TestHBaseTable extends QueryTestCaseBase {
} catch (Exception e) {
assertTrue(e.getMessage().indexOf("'columns' property is required") >= 0);
}
-
- try {
- executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text) " +
- "USING hbase " +
- "WITH ('table'='hbase_table', 'columns'='col1:,col2:')").close();
-
- fail("hbase table must have 'hbase.zookeeper.quorum' meta");
- } catch (Exception e) {
- assertTrue(e.getMessage().indexOf("HBase mapped table") >= 0);
- }
- }
-
-
- @Test
- public void testVerifyCreateHBaseTableWithHBaseConfiguration() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
- try {
- testingCluster.getHBaseUtil().getConf().set(HConstants.ZOOKEEPER_QUORUM, hostName);
-
- executeString("CREATE TABLE hbase_mapped_table3 (col1 text, col2 text) " +
- "USING hbase " +
- "WITH ('table'='hbase_mapped_table3', 'columns'='col1:,col2:')").close();
-
- testingCluster.getHBaseUtil().getConf().set(HConstants.ZOOKEEPER_QUORUM, null);
- } catch (Exception e) {
- assertFalse(e.getMessage().indexOf("HBase mapped table") >= 0);
- }
}
@Test
public void testCreateHBaseTable() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
@@ -166,10 +138,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testCreateNotExistsExternalHBaseTable() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
try {
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table1 (col1 text, col2 text, col3 text, col4 text) " +
"USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:a,col3:,col2:b', " +
@@ -183,10 +151,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testCreateRowFieldWithNonText() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
try {
executeString("CREATE TABLE hbase_mapped_table2 (rk1 int4, rk2 text, col3 text, col4 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'='0:key#b,1:key,col3:,col2:b', " +
@@ -207,10 +171,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
hTableDesc.addFamily(new HColumnDescriptor("col3"));
testingCluster.getHBaseUtil().createTable(hTableDesc);
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " +
"USING hbase WITH ('table'='external_hbase_table_not_purge', 'columns'=':key,col1:a,col2:,col3:b', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
@@ -238,10 +198,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
hTableDesc.addFamily(new HColumnDescriptor("col3"));
testingCluster.getHBaseUtil().createTable(hTableDesc);
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " +
"USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
@@ -281,10 +237,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
hTableDesc.addFamily(new HColumnDescriptor("col3"));
testingCluster.getHBaseUtil().createTable(hTableDesc);
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk int8, col1 text, col2 text, col3 int4)\n " +
"USING hbase WITH ('table'='external_hbase_table', 'columns'=':key#b,col1:a,col2:,col3:b#b', \n" +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "', \n" +
@@ -337,10 +289,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
hTableDesc.addFamily(new HColumnDescriptor("col3"));
testingCluster.getHBaseUtil().createTable(hTableDesc);
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, col2_key text, col2_value text, col3 text) " +
"USING hbase WITH ('table'='external_hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " +
"'hbase.rowkey.delimiter'='_', " +
@@ -378,10 +326,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
hTableDesc.addFamily(new HColumnDescriptor("col3"));
testingCluster.getHBaseUtil().createTable(hTableDesc);
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE EXTERNAL TABLE external_hbase_mapped_table (rk1 text, rk2 text, col3 text) " +
"USING hbase WITH ('table'='external_hbase_table', 'columns'='0:key,1:key,col3:a', " +
"'hbase.rowkey.delimiter'='_', " +
@@ -412,10 +356,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testIndexPredication() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
@@ -464,10 +404,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testCompositeRowIndexPredication() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, rk2 text, col1 text, col2 text, col3 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a,col2:,col3:b', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
@@ -475,7 +411,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
"'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close();
-
assertTableExists("hbase_mapped_table");
HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf());
hAdmin.tableExists("hbase_table");
@@ -623,17 +558,12 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testNonForwardQuery() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:#b', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
"'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close();
-
assertTableExists("hbase_mapped_table");
HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf());
HTable htable = null;
@@ -668,17 +598,12 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testJoin() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int8) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
"'" + HConstants.ZOOKEEPER_CLIENT_PORT + "'='" + zkPort + "')").close();
-
assertTableExists("hbase_mapped_table");
HBaseAdmin hAdmin = new HBaseAdmin(testingCluster.getHBaseUtil().getConf());
HTable htable = null;
@@ -715,10 +640,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertInto() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int4) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
@@ -761,10 +682,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoMultiRegion() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
@@ -823,10 +740,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoMultiRegion2() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " +
"'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " +
@@ -884,10 +797,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoMultiRegionWithSplitFile() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
String splitFilePath = currentDatasetPath + "/splits.data";
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " +
@@ -948,10 +857,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoMultiRegionMultiRowFields() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a', " +
"'hbase.split.rowkeys'='001,002,003,004,005,006,007,008,009', " +
@@ -1012,10 +917,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoBinaryMultiRegion() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk int4, col1 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key#b,col1:a', " +
"'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " +
@@ -1073,10 +974,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoColumnKeyValue() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col2_key text, col2_value text, col3 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col2:key:,col2:value:,col3:', " +
"'hbase.rowkey.delimiter'='_', " +
@@ -1168,10 +1065,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoDifferentType() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a', " +
"'hbase.split.rowkeys'='1,2,3,4,5,6,7,8,9', " +
@@ -1209,10 +1102,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoRowField() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk1 text, rk2 text, col1 text, col2 text, col3 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'='0:key,1:key,col1:a,col2:,col3:b', " +
"'hbase.rowkey.delimiter'='_', " +
@@ -1257,10 +1146,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testCATS() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
// create test table
KeyValueSet tableOptions = new KeyValueSet();
tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
@@ -1319,10 +1204,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoUsingPut() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text, col3 int4) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:,col3:b#b', " +
"'" + HConstants.ZOOKEEPER_QUORUM + "'='" + hostName + "'," +
@@ -1372,10 +1253,6 @@ public class TestHBaseTable extends QueryTestCaseBase {
@Test
public void testInsertIntoLocation() throws Exception {
- String hostName = InetAddress.getLocalHost().getHostName();
- String zkPort = testingCluster.getHBaseUtil().getConf().get(HConstants.ZOOKEEPER_CLIENT_PORT);
- assertNotNull(zkPort);
-
executeString("CREATE TABLE hbase_mapped_table (rk text, col1 text, col2 text) " +
"USING hbase WITH ('table'='hbase_table', 'columns'=':key,col1:a,col2:', " +
"'hbase.split.rowkeys'='010,040,060,080', " +
http://git-wip-us.apache.org/repos/asf/tajo/blob/c429c971/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index f10f393..cc611d5 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -226,16 +226,37 @@ HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/"
##############################################################################
##############################################################################
-# Set HBase CLASSPATH
+# HBase Configuration Start
##############################################################################
-if [ "$HBASE_HOME" != "" ]; then
- for f in ${HBASE_HOME}/lib/hbase-*.jar; do
+
+HBASE_CONF=$HBASE_HOME/conf
+
+if [ -d ${HBASE_CONF} ]; then
+ CLASSPATH=${HBASE_CONF}:${CLASSPATH}
+fi
+
+HBASE_LIB=$HBASE_HOME/lib
+
+if [ -d ${HBASE_LIB} ]; then
+
+ for f in ${HBASE_LIB}/hbase-client-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
- for f in ${HBASE_HOME}/lib/htrace-*.jar; do
+
+ for f in ${HBASE_LIB}/hbase-common-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ for f in ${HBASE_LIB}/hbase-protocol-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ for f in ${HBASE_LIB}/htrace-core-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
+
fi
+
##############################################################################
# HBase Configuration End
##############################################################################
@@ -289,31 +310,6 @@ fi
##############################################################################
-# Find and Set HBase CLASSPATH
-##############################################################################
-
-HBASE_CONF=$HBASE_HOME/conf/hbase-site.xml
-
-if [ -d ${HBASE_CONF} ]; then
- CLASSPATH=${HBASE_CONF}:${CLASSPATH}
-fi
-
-
-HBASE_LIB=$HBASE_HOME/lib
-
-if [ -d ${HBASE_LIB} ]; then
-
- for f in ${HBASE_LIB}/hbase-common-*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
- done
-fi
-
-##############################################################################
-# Find and Set HBase CLASSPATH
-##############################################################################
-
-
-##############################################################################
# Find and Set Tajo CLASSPATH
##############################################################################
http://git-wip-us.apache.org/repos/asf/tajo/blob/c429c971/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index c51e31c..59d1b48 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -299,12 +299,12 @@ public class HBaseStorageManager extends StorageManager {
* @throws java.io.IOException
*/
public static Configuration getHBaseConfiguration(Configuration conf, TableMeta tableMeta) throws IOException {
- boolean hasZkQuorum = tableMeta.containsOption(HBaseStorageConstants.META_ZK_QUORUM_KEY);
- String zkQuorum = null;
- if (hasZkQuorum) {
+ Configuration hbaseConf = (conf == null) ? HBaseConfiguration.create() : HBaseConfiguration.create(conf);
+
+ String zkQuorum = hbaseConf.get(HConstants.ZOOKEEPER_QUORUM);
+ if (tableMeta.containsOption(HBaseStorageConstants.META_ZK_QUORUM_KEY)) {
zkQuorum = tableMeta.getOption(HBaseStorageConstants.META_ZK_QUORUM_KEY, "");
- } else {
- zkQuorum = conf.get(HBaseStorageConstants.META_ZK_QUORUM_KEY, "");
+ hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum);
}
if (zkQuorum == null || zkQuorum.trim().isEmpty()) {
@@ -312,9 +312,6 @@ public class HBaseStorageManager extends StorageManager {
HBaseStorageConstants.META_ZK_QUORUM_KEY + "' attribute.");
}
- Configuration hbaseConf = (conf == null) ? HBaseConfiguration.create() : HBaseConfiguration.create(conf);
- hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zkQuorum);
-
for (Map.Entry<String, String> eachOption: tableMeta.getOptions().getAllKeyValus().entrySet()) {
String key = eachOption.getKey();
if (key.startsWith(HConstants.ZK_CFG_PROPERTY_PREFIX)) {