You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/02/02 13:29:12 UTC
[07/10] tajo git commit: TAJO-1323: Cleanup the unstable test case.
(jinho)
TAJO-1323: Cleanup the unstable test case. (jinho)
Closes #368
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd73074f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd73074f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd73074f
Branch: refs/heads/index_support
Commit: fd73074f2198f92516a3eb8f5e786d31c1c071a5
Parents: e656ee2
Author: jhkim <jh...@apache.org>
Authored: Mon Feb 2 14:56:14 2015 +0900
Committer: jhkim <jh...@apache.org>
Committed: Mon Feb 2 14:56:14 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 3 +-
.../java/org/apache/tajo/querymaster/Stage.java | 23 ++++-
.../org/apache/tajo/TajoTestingCluster.java | 96 ++++++++------------
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 42 +++------
.../tajo/engine/query/TestInsertQuery.java | 3 +-
.../apache/tajo/querymaster/TestKillQuery.java | 20 +++-
.../TestTajoCli/testNonForwardQueryPause.result | 3 +-
8 files changed, 98 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 4a3715b..8466a38 100644
--- a/CHANGES
+++ b/CHANGES
@@ -327,6 +327,8 @@ Release 0.10.0 - unreleased
TASKS
+ TAJO-1323: Cleanup the unstable test case. (jinho)
+
TAJO-1295: Remove legacy worker.dataserver package and its unit tests.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
index 17c94b9..5cbe77b 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
@@ -37,6 +37,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
private boolean printPause;
private boolean printErrorTrace;
private String nullChar;
+ public static char QUIT_COMMAND = 'q';
@Override
public void init(TajoCli.TajoCliContext context) {
@@ -123,7 +124,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
}
sout.flush();
if (sin != null) {
- if (sin.read() == 'q') {
+ if (sin.read() == QUIT_COMMAND) {
endOfTuple = false;
sout.println();
break;
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 208d4a6..5673d5b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -185,7 +185,7 @@ public class Stage implements EventHandler<StageEvent> {
// Transitions from KILL_WAIT state
.addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
- StageEventType.SQ_CONTAINER_ALLOCATED,
+ EnumSet.of(StageEventType.SQ_START, StageEventType.SQ_CONTAINER_ALLOCATED),
CONTAINERS_CANCEL_TRANSITION)
.addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
EnumSet.of(StageEventType.SQ_KILL), new KillTasksTransition())
@@ -414,6 +414,18 @@ public class Stage implements EventHandler<StageEvent> {
return totalScheduledObjectsCount;
}
+ public int getKilledObjectCount() {
+ return killedObjectCount;
+ }
+
+ public int getFailedObjectCount() {
+ return failedObjectCount;
+ }
+
+ public int getCompletedTaskCount() {
+ return completedTaskCount;
+ }
+
public ExecutionBlock getBlock() {
return block;
}
@@ -793,7 +805,14 @@ public class Stage implements EventHandler<StageEvent> {
stage.taskScheduler.start();
allocateContainers(stage);
} else {
- stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_KILL));
+ /* all tasks are killed before stage are inited */
+ if (stage.getTotalScheduledObjectsCount() == stage.getCompletedTaskCount()) {
+ stage.eventHandler.handle(
+ new StageEvent(stage.getId(), StageEventType.SQ_STAGE_COMPLETED));
+ } else {
+ stage.eventHandler.handle(
+ new StageEvent(stage.getId(), StageEventType.SQ_KILL));
+ }
}
}
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 8714fc4..0d3d660 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -77,17 +77,13 @@ public class TajoTestingCluster {
private TajoMaster tajoMaster;
private List<TajoWorker> tajoWorkers = new ArrayList<TajoWorker>();
private boolean standbyWorkerMode = false;
+ private boolean isDFSRunning = false;
+ private boolean isTajoClusterRunning = false;
+ private boolean isCatalogServerRunning = false;
- // If non-null, then already a cluster running.
private File clusterTestBuildDir = null;
/**
- * System property key to get test directory value.
- * Name is as it is because mini dfs has hard-codings to put test data here.
- */
- public static final String TEST_DIRECTORY_KEY = MiniDFSCluster.PROP_TEST_BUILD_DATA;
-
- /**
* Default parent directory for test output.
*/
public static final String DEFAULT_TEST_DIRECTORY = "target/test-data";
@@ -111,6 +107,7 @@ public class TajoTestingCluster {
this.conf = new TajoConf();
this.conf.setBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE, masterHaEMode);
+ initTestDir();
setTestingFlagProperties();
initPropertiesAndConfigs();
}
@@ -177,22 +174,19 @@ public class TajoTestingCluster {
return this.conf;
}
- public void initTestDir() {
- if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
- clusterTestBuildDir = setupClusterTestBuildDir();
- System.setProperty(TEST_DIRECTORY_KEY,
- clusterTestBuildDir.getAbsolutePath());
- }
- }
+ public void initTestDir() {
+ if (clusterTestBuildDir == null) {
+ clusterTestBuildDir = setupClusterTestBuildDir();
+ }
+ }
/**
* @return Where to write test data on local filesystem; usually
* {@link #DEFAULT_TEST_DIRECTORY}
* @see #setupClusterTestBuildDir()
*/
- public static File getTestDir() {
- return new File(System.getProperty(TEST_DIRECTORY_KEY,
- DEFAULT_TEST_DIRECTORY));
+ public File getTestDir() {
+ return clusterTestBuildDir;
}
/**
@@ -202,10 +196,10 @@ public class TajoTestingCluster {
* @see #setupClusterTestBuildDir()
*/
public static File getTestDir(final String subdirName) {
- return new File(getTestDir(), subdirName);
+ return new File(new File(DEFAULT_TEST_DIRECTORY), subdirName);
}
- public File setupClusterTestBuildDir() {
+ public static File setupClusterTestBuildDir() {
String randomStr = UUID.randomUUID().toString();
String dirStr = getTestDir(randomStr).toString();
File dir = new File(dirStr).getAbsoluteFile();
@@ -243,9 +237,6 @@ public class TajoTestingCluster {
File dir,
final String hosts[])
throws IOException {
- if (dir == null) {
- dir = setupClusterTestBuildDir();
- }
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString());
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
@@ -263,7 +254,7 @@ public class TajoTestingCluster {
this.defaultFS = this.dfsCluster.getFileSystem();
this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
-
+ isDFSRunning = true;
return this.dfsCluster;
}
@@ -300,22 +291,20 @@ public class TajoTestingCluster {
// Catalog Section
////////////////////////////////////////////////////////
public MiniCatalogServer startCatalogCluster() throws Exception {
- TajoConf c = getConfiguration();
+ if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running");
- if(clusterTestBuildDir == null) {
- clusterTestBuildDir = setupClusterTestBuildDir();
- }
+ TajoConf c = getConfiguration();
conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore");
conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath() + "/db");
- LOG.info("Apache Derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI));
+ LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI));
conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0");
catalogServer = new MiniCatalogServer(conf);
CatalogServer catServer = catalogServer.getCatalogServer();
InetSocketAddress sockAddr = catServer.getBindAddress();
c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr));
-
+ isCatalogServerRunning = true;
return this.catalogServer;
}
@@ -323,6 +312,7 @@ public class TajoTestingCluster {
if (catalogServer != null) {
this.catalogServer.shutdown();
}
+ isCatalogServerRunning = false;
}
public MiniCatalogServer getMiniCatalogCluster() {
@@ -352,10 +342,10 @@ public class TajoTestingCluster {
c.setVar(ConfVars.ROOT_DIR,
getMiniDFSCluster().getFileSystem().getUri() + "/tajo");
} else {
- c.setVar(ConfVars.ROOT_DIR, clusterTestBuildDir.getAbsolutePath() + "/tajo");
+ c.setVar(ConfVars.ROOT_DIR, testBuildDir.getAbsolutePath() + "/tajo");
}
- setupCatalogForTesting(c, clusterTestBuildDir);
+ setupCatalogForTesting(c, testBuildDir);
tajoMaster = new TajoMaster();
tajoMaster.init(c);
@@ -374,6 +364,7 @@ public class TajoTestingCluster {
if(standbyWorkerMode) {
startTajoWorkers(numSlaves);
}
+ isTajoClusterRunning = true;
LOG.info("Mini Tajo cluster is up");
LOG.info("====================================================================================");
LOG.info("= MiniTajoCluster starts up =");
@@ -473,8 +464,8 @@ public class TajoTestingCluster {
/**
* @throws java.io.IOException If a cluster -- dfs or engine -- already running.
*/
- void isRunningCluster(String passedBuildPath) throws IOException {
- if (this.clusterTestBuildDir == null || passedBuildPath != null) return;
+ void isRunningCluster() throws IOException {
+ if (!isTajoClusterRunning && !isCatalogServerRunning && !isDFSRunning) return;
throw new IOException("Cluster already running at " +
this.clusterTestBuildDir);
}
@@ -501,19 +492,13 @@ public class TajoTestingCluster {
LOG.info("Starting up minicluster with 1 master(s) and " +
numSlaves + " worker(s) and " + numDataNodes + " datanode(s)");
- // If we already put up a cluster, fail.
- String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
- isRunningCluster(testBuildPath);
- if (testBuildPath != null) {
- LOG.info("Using passed path: " + testBuildPath);
+ // If we already bring up the cluster, fail.
+ isRunningCluster();
+ if (clusterTestBuildDir != null) {
+ LOG.info("Using passed path: " + clusterTestBuildDir);
}
- // Make a new random dir to home everything in. Set it as system property.
- // minidfs reads home from system property.
- this.clusterTestBuildDir = testBuildPath == null?
- setupClusterTestBuildDir() : new File(testBuildPath);
-
- startMiniDFSCluster(numDataNodes, setupClusterTestBuildDir(), dataNodeHosts);
+ startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts);
this.dfsCluster.waitClusterUp();
hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir);
@@ -559,20 +544,11 @@ public class TajoTestingCluster {
}
public void startMiniClusterInLocal(final int numSlaves) throws Exception {
- // If we already put up a cluster, fail.
- String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
- isRunningCluster(testBuildPath);
- if (testBuildPath != null) {
- LOG.info("Using passed path: " + testBuildPath);
- }
-
- // Make a new random dir to home everything in. Set it as system property.
- // minidfs reads home from system property.
- this.clusterTestBuildDir = testBuildPath == null?
- setupClusterTestBuildDir() : new File(testBuildPath);
+ isRunningCluster();
- System.setProperty(TEST_DIRECTORY_KEY,
- this.clusterTestBuildDir.getAbsolutePath());
+ if (clusterTestBuildDir != null) {
+ LOG.info("Using passed path: " + clusterTestBuildDir);
+ }
startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, true);
}
@@ -592,6 +568,7 @@ public class TajoTestingCluster {
if(this.catalogServer != null) {
shutdownCatalogCluster();
+ isCatalogServerRunning = false;
}
if(this.yarnCluster != null) {
@@ -612,6 +589,7 @@ public class TajoTestingCluster {
} catch (IOException e) {
System.err.println("error closing file system: " + e);
}
+ isDFSRunning = false;
}
if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
@@ -630,6 +608,7 @@ public class TajoTestingCluster {
}
LOG.info("Minicluster is down");
+ isTajoClusterRunning = false;
}
public static TajoClient newTajoClient() throws Exception {
@@ -803,7 +782,8 @@ public class TajoTestingCluster {
} catch (InterruptedException e) {
}
if (++i > 200) {
- throw new IOException("Timed out waiting");
+ throw new IOException("Timed out waiting. expected: " + expected +
+ ", actual: " + query != null ? String.valueOf(query.getSynchronizedState()) : String.valueOf(query));
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
index aff1677..e014b52 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
@@ -38,9 +38,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.PrintWriter;
+import java.io.*;
import java.net.URL;
import static org.junit.Assert.*;
@@ -77,7 +75,8 @@ public class TestTajoCli {
}
@After
- public void tearDown() {
+ public void tearDown() throws IOException {
+ out.close();
if (tajoCli != null) {
tajoCli.close();
}
@@ -350,38 +349,27 @@ public class TestTajoCli {
assertOutputResult(new String(out.toByteArray()));
}
- @Test
+ @Test(timeout = 3000)
public void testNonForwardQueryPause() throws Exception {
final String sql = "select * from default.lineitem";
+ TajoCli cli = null;
try {
TableDesc tableDesc = cluster.getMaster().getCatalog().getTableDesc("default", "lineitem");
assertNotNull(tableDesc);
assertEquals(0L, tableDesc.getStats().getNumRows().longValue());
- setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "2");
- setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
- Thread t = new Thread() {
- public void run() {
- try {
- tajoCli.executeScript(sql);
- } catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
- };
- t.start();
+
+ InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND});
+ cli = new TajoCli(cluster.getConfiguration(), new String[]{}, testInput, out);
+ setVar(cli, SessionVars.CLI_PAGE_ROWS, "2");
+ setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+
+ cli.executeScript(sql);
+
String consoleResult;
- while (true) {
- Thread.sleep(3 * 1000);
- consoleResult = new String(out.toByteArray());
- if (consoleResult.indexOf("row") >= 0) {
- t.interrupt();
- break;
- }
- }
+ consoleResult = new String(out.toByteArray());
assertOutputResult(consoleResult);
} finally {
- setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "100");
+ cli.close();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index cc7dced..0799d22 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -660,12 +660,11 @@ public class TestInsertQuery extends QueryTestCaseBase {
@Test
public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception {
- String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery");
+ String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery2");
ResultSet res = executeString("create table " + tableName +" (col1 int4, col2 float4, col3 text)");
res.close();
CatalogService catalog = testingCluster.getMaster().getCatalog();
assertTrue(catalog.existsTable(getCurrentDatabase(), tableName));
-
res = executeString("insert overwrite into " + tableName + " (col1, col3) select 1::INT4, 'test';");
res.close();
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
index 42ad8da..0574bea 100644
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
@@ -127,12 +127,24 @@ public class TestKillQuery {
Query q = queryMasterTask.getQuery();
q.handle(new QueryEvent(queryId, QueryEventType.KILL));
- try{
+ try {
cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50);
- } finally {
assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState());
+ } catch (Exception e) {
+ e.printStackTrace();
+ if (stage != null) {
+ System.err.println(String.format("Stage: [%s] (Total: %d, Complete: %d, Success: %d, Killed: %d, Failed: %d)",
+ stage.getId().toString(),
+ stage.getTotalScheduledObjectsCount(),
+ stage.getCompletedTaskCount(),
+ stage.getSucceededObjectCount(),
+ stage.getKilledObjectCount(),
+ stage.getFailedObjectCount()));
+ }
+ throw e;
+ } finally {
+ queryMasterTask.stop();
}
- queryMasterTask.stop();
}
@Test
@@ -145,6 +157,8 @@ public class TestKillQuery {
QueryMasterTask qmt = cluster.getQueryMasterTask(queryId);
Query query = qmt.getQuery();
+ // wait for a stage created
+ cluster.waitForQueryState(query, TajoProtos.QueryState.QUERY_RUNNING, 10);
query.handle(new QueryEvent(queryId, QueryEventType.KILL));
try{
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
index e9485d0..d4ba604 100644
--- a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
@@ -2,4 +2,5 @@ l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice
-------------------------------
1, 1, 7706, 1, 17.0, 21168.23, 0.04, 0.02, N, O, 1996-03-13, 1996-02-12, 1996-03-22, DELIVER IN PERSON, TRUCK, egular courts above the
1, 1, 7311, 2, 36.0, 45983.16, 0.09, 0.06, N, O, 1996-04-12, 1996-02-28, 1996-04-20, TAKE BACK RETURN, MAIL, ly final dependencies: slyly bold
-(2 rows, continue... 'q' is quit)
\ No newline at end of file
+(2 rows, continue... 'q' is quit)
+(unknown row number, , 604 B selected)
\ No newline at end of file