You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/02/03 09:01:46 UTC
[1/3] tajo git commit: TAJO-1324: Remove warehouse directory
rewriting in Unit Test. (jinho)
Repository: tajo
Updated Branches:
refs/heads/index_support 291817c83 -> 1c1dfa0ed
TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho)
Closes #370
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9ad117f0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9ad117f0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9ad117f0
Branch: refs/heads/index_support
Commit: 9ad117f07b30874685e58e28b76f8d0a2650b0f5
Parents: e9cd25a
Author: jhkim <jh...@apache.org>
Authored: Tue Feb 3 15:14:33 2015 +0900
Committer: jhkim <jh...@apache.org>
Committed: Tue Feb 3 15:14:33 2015 +0900
----------------------------------------------------------------------
.travis.yml | 2 +-
CHANGES | 2 ++
.../java/org/apache/tajo/master/TajoMaster.java | 9 ++-------
.../java/org/apache/tajo/BackendTestingUtil.java | 5 ++---
.../org/apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../java/org/apache/tajo/TajoTestingCluster.java | 12 ++++++++----
.../engine/planner/physical/TestBNLJoinExec.java | 2 --
.../engine/planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 2 --
.../physical/TestFullOuterHashJoinExec.java | 2 --
.../physical/TestFullOuterMergeJoinExec.java | 2 --
.../planner/physical/TestHashAntiJoinExec.java | 2 --
.../engine/planner/physical/TestHashJoinExec.java | 2 --
.../planner/physical/TestHashSemiJoinExec.java | 2 --
.../physical/TestLeftOuterHashJoinExec.java | 2 --
.../planner/physical/TestLeftOuterNLJoinExec.java | 2 --
.../planner/physical/TestMergeJoinExec.java | 11 ++++-------
.../engine/planner/physical/TestNLJoinExec.java | 2 --
.../planner/physical/TestPhysicalPlanner.java | 2 +-
.../physical/TestProgressExternalSortExec.java | 2 --
.../physical/TestRightOuterHashJoinExec.java | 2 --
.../physical/TestRightOuterMergeJoinExec.java | 2 --
.../engine/planner/physical/TestSortExec.java | 2 +-
.../apache/tajo/engine/query/TestCreateTable.java | 2 +-
.../java/org/apache/tajo/storage/TestRowFile.java | 3 +--
.../org/apache/tajo/storage/StorageManager.java | 18 +-----------------
.../tajo/storage/HashShuffleAppenderManager.java | 4 ++--
.../tajo/storage/TestFileStorageManager.java | 4 ++--
.../org/apache/tajo/storage/TestMergeScanner.java | 2 +-
29 files changed, 31 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 0a6eec0..ab16f1d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,4 +37,4 @@ notifications:
install: ./dev-support/travis-install-dependencies.sh
script:
- mvn clean install -Pparallel-test,hcatalog-0.12.0 -DLOG_LEVEL=WARN -Dmaven.fork.count=3
+ mvn clean install -Pparallel-test,hcatalog-0.12.0 -DLOG_LEVEL=WARN -Dmaven.fork.count=2
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6b568e3..47eaf9c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -176,6 +176,8 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho)
+
TAJO-1321: Cli prints wrong response time. (jihoon)
TAJO-1313: Tajo-dump creates DDLs for information_schema tables.
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index a11606f..aee8a74 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -52,7 +52,6 @@ import org.apache.tajo.rule.SelfDiagnosisRuleSession;
import org.apache.tajo.service.ServiceTracker;
import org.apache.tajo.service.ServiceTrackerFactory;
import org.apache.tajo.session.SessionManager;
-import org.apache.tajo.storage.FileStorageManager;
import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.util.*;
import org.apache.tajo.util.history.HistoryReader;
@@ -110,7 +109,7 @@ public class TajoMaster extends CompositeService {
private CatalogServer catalogServer;
private CatalogService catalog;
- private FileStorageManager storeManager;
+ private StorageManager storeManager;
private GlobalEngine globalEngine;
private AsyncDispatcher dispatcher;
private TajoMasterClientService tajoMasterClientService;
@@ -171,7 +170,7 @@ public class TajoMaster extends CompositeService {
// check the system directory and create if they are not created.
checkAndInitializeSystemDirectories();
diagnoseTajoMaster();
- this.storeManager = (FileStorageManager)StorageManager.getFileStorageManager(systemConf, null);
+ this.storeManager = StorageManager.getFileStorageManager(systemConf);
catalogServer = new CatalogServer(FunctionLoader.load());
addIfService(catalogServer);
@@ -410,10 +409,6 @@ public class TajoMaster extends CompositeService {
return this.catalogServer;
}
- public FileStorageManager getStorageManager() {
- return this.storeManager;
- }
-
public class MasterContext {
private final TajoConf conf;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
index fb98be2..308ed28 100644
--- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java
@@ -47,12 +47,11 @@ public class BackendTestingUtil {
public static void writeTmpTable(TajoConf conf, Path tablePath)
throws IOException {
- FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, tablePath);
- FileSystem fs = sm.getFileSystem();
-
+ FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(conf);
Appender appender;
Path filePath = new Path(tablePath, "table.csv");
+ FileSystem fs = filePath.getFileSystem(conf);
if (fs.exists(tablePath)) {
fs.delete(tablePath, true);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index e464446..7b5e610 100644
--- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -106,7 +106,7 @@ public class LocalTajoTestingUtility {
client = new TajoClientImpl(conf);
FileSystem fs = util.getDefaultFileSystem();
- Path rootDir = util.getMaster().getStorageManager().getWarehouseDir();
+ Path rootDir = TajoConf.getWarehouseDir(conf);
fs.mkdirs(rootDir);
for (int i = 0; i < tablepaths.length; i++) {
Path localPath = new Path(tablepaths[i]);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index a93d7f0..43bd52a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -158,6 +158,12 @@ public class TajoTestingCluster {
conf.setIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
conf.setIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ // Resource allocator
+ conf.setIntVar(ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM, 2);
+
+ // Memory cache termination
+ conf.setIntVar(ConfVars.WORKER_HISTORY_EXPIRE_PERIOD, 1);
+
this.standbyWorkerMode = conf.getVar(ConfVars.RESOURCE_MANAGER_CLASS)
.indexOf(TajoWorkerResourceManager.class.getName()) >= 0;
@@ -635,8 +641,7 @@ public class TajoTestingCluster {
TajoTestingCluster util = TpchTestBase.getInstance().getTestingCluster();
FileSystem fs = util.getDefaultFileSystem();
- Path rootDir = util.getMaster().
- getStorageManager().getWarehouseDir();
+ Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration());
fs.mkdirs(rootDir);
for (int i = 0; i < names.length; i++) {
createTable(names[i], schemas[i], tableOption, tables[i]);
@@ -688,8 +693,7 @@ public class TajoTestingCluster {
TajoClient client = new TajoClientImpl(conf);
try {
FileSystem fs = util.getDefaultFileSystem();
- Path rootDir = util.getMaster().
- getStorageManager().getWarehouseDir();
+ Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration());
if (!fs.exists(rootDir)) {
fs.mkdirs(rootDir);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index 0bfcc22..842a120 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -62,7 +62,6 @@ public class TestBNLJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private static int OUTER_TUPLE_NUM = 1000;
@@ -79,7 +78,6 @@ public class TestBNLJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index 2ddb4ef..b22a87e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -91,7 +91,7 @@ public class TestBSTIndexExec {
Path workDir = CommonTestingUtil.getTestDir();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir);
+ sm = (FileStorageManager)StorageManager.getFileStorageManager(conf);
idxPath = new Path(workDir, "test.idx");
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index 2047e41..5d9d46d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -59,7 +59,6 @@ public class TestExternalSortExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private FileStorageManager sm;
private Path testDir;
private final int numTuple = 100000;
@@ -76,7 +75,6 @@ public class TestExternalSortExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index 04117c1..780e698 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -61,7 +61,6 @@ public class TestFullOuterHashJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private FileStorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -84,7 +83,6 @@ public class TestFullOuterHashJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 5d2991b..ef6f1ce 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -63,7 +63,6 @@ public class TestFullOuterMergeJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -89,7 +88,6 @@ public class TestFullOuterMergeJoinExec {
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 6085f3d..64efa28 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -61,7 +61,6 @@ public class TestHashAntiJoinExec {
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
private LogicalOptimizer optimizer;
- private StorageManager sm;
private Path testDir;
private TableDesc employee;
@@ -76,7 +75,6 @@ public class TestHashAntiJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema employeeSchema = new Schema();
employeeSchema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index 50d6ca1..578b586 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -62,7 +62,6 @@ public class TestHashJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -78,7 +77,6 @@ public class TestHashJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema employeeSchema = new Schema();
employeeSchema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 978f635..42a018b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -62,7 +62,6 @@ public class TestHashSemiJoinExec {
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
private LogicalOptimizer optimizer;
- private StorageManager sm;
private Path testDir;
private TableDesc employee;
@@ -77,7 +76,6 @@ public class TestHashSemiJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema employeeSchema = new Schema();
employeeSchema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index 593c28b..5747e58 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -62,7 +62,6 @@ public class TestLeftOuterHashJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -85,7 +84,6 @@ public class TestLeftOuterHashJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index 72f4ab6..ab4b881 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -60,7 +60,6 @@ public class TestLeftOuterNLJoinExec {
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
private QueryContext defaultContext;
- private StorageManager sm;
private Path testDir;
private TableDesc dep3;
@@ -81,7 +80,6 @@ public class TestLeftOuterNLJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 65f7799..1cc4e63 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -18,7 +18,6 @@
package org.apache.tajo.engine.planner.physical;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.LocalTajoTestingUtility;
import org.apache.tajo.TajoConstants;
@@ -31,16 +30,17 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.engine.planner.*;
+import org.apache.tajo.engine.planner.PhysicalPlanner;
+import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -64,7 +64,6 @@ public class TestMergeJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private TableDesc employee;
private TableDesc people;
@@ -78,8 +77,6 @@ public class TestMergeJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- FileSystem fs = testDir.getFileSystem(conf);
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema employeeSchema = new Schema();
employeeSchema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index ffbf34a..240e7fd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -60,7 +60,6 @@ public class TestNLJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private TableDesc employee;
@@ -76,7 +75,6 @@ public class TestNLJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 0c8fa36..e2216b9 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -99,7 +99,7 @@ public class TestPhysicalPlanner {
util.startCatalogCluster();
conf = util.getConfiguration();
testDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestPhysicalPlanner");
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, testDir);
+ sm = (FileStorageManager)StorageManager.getFileStorageManager(conf);
catalog = util.getMiniCatalogCluster().getCatalog();
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index 29fff7c..161b0d8 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -62,7 +62,6 @@ public class TestProgressExternalSortExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private final int numTuple = 100000;
@@ -80,7 +79,6 @@ public class TestProgressExternalSortExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString());
- sm = StorageManager.getFileStorageManager(conf, testDir);
Schema schema = new Schema();
schema.addColumn("managerid", TajoDataTypes.Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index 7578690..06ab6c8 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -59,7 +59,6 @@ public class TestRightOuterHashJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -80,7 +79,6 @@ public class TestRightOuterHashJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 8a812ea..30fd999 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -61,7 +61,6 @@ public class TestRightOuterMergeJoinExec {
private CatalogService catalog;
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
- private StorageManager sm;
private Path testDir;
private QueryContext defaultContext;
@@ -87,7 +86,6 @@ public class TestRightOuterMergeJoinExec {
catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString());
catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
conf = util.getConfiguration();
- sm = StorageManager.getFileStorageManager(conf, testDir);
//----------------- dep3 ------------------------------
// dep_id | dep_name | loc_id
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index a42fbac..f2c2b1a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -70,7 +70,7 @@ public class TestSortExec {
util = TpchTestBase.getInstance().getTestingCluster();
catalog = util.getMaster().getCatalog();
workDir = CommonTestingUtil.getTestDir(TEST_PATH);
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir);
+ sm = (FileStorageManager)StorageManager.getFileStorageManager(conf);
Schema schema = new Schema();
schema.addColumn("managerid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index 18f55ff..21b3910 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -105,8 +105,8 @@ public class TestCreateTable extends QueryTestCaseBase {
// checking the existence of the table directory and validating the path
- FileSystem fs = testingCluster.getMaster().getStorageManager().getFileSystem();
Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
+ FileSystem fs = warehouseDir.getFileSystem(testingCluster.getConfiguration());
assertTrue(fs.exists(new Path(oldTableDesc.getPath())));
assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
new Path(oldTableDesc.getPath()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
index f36ff24..859a1f2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -32,7 +32,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.storage.fragment.FileFragment;
@@ -70,7 +69,7 @@ public class TestRowFile {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE);
FileStorageManager sm =
- (FileStorageManager)StorageManager.getFileStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR)));
+ (FileStorageManager)StorageManager.getFileStorageManager(conf);
Path tablePath = new Path("/test");
Path metaPath = new Path(tablePath, ".meta");
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index d929591..ce963c8 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -271,23 +271,7 @@ public abstract class StorageManager {
* @throws java.io.IOException
*/
public static StorageManager getFileStorageManager(TajoConf tajoConf) throws IOException {
- return getFileStorageManager(tajoConf, null);
- }
-
- /**
- * Returns FileStorageManager instance and sets WAREHOUSE_DIR property in tajoConf with warehousePath parameter.
- *
- * @param tajoConf Tajo system property.
- * @param warehousePath The warehouse directory to be set in the tajoConf.
- * @return
- * @throws java.io.IOException
- */
- public static StorageManager getFileStorageManager(TajoConf tajoConf, Path warehousePath) throws IOException {
- TajoConf copiedConf = new TajoConf(tajoConf);
- if (warehousePath != null) {
- copiedConf.setVar(ConfVars.WAREHOUSE_DIR, warehousePath.toUri().toString());
- }
- return getStorageManager(copiedConf, StoreType.CSV);
+ return getStorageManager(tajoConf, StoreType.CSV);
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
index dededc6..466c6c7 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
@@ -84,8 +84,8 @@ public class HashShuffleAppenderManager {
if (!fs.exists(dataFile.getParent())) {
fs.mkdirs(dataFile.getParent());
}
- FileAppender appender = (FileAppender)((FileStorageManager)StorageManager.getFileStorageManager(
- tajoConf, null)).getAppender(meta, outSchema, dataFile);
+ FileAppender appender = (FileAppender)((FileStorageManager)StorageManager.getFileStorageManager(tajoConf))
+ .getAppender(meta, outSchema, dataFile);
appender.enableStats();
appender.init();
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
index c4df8d7..4b39cdd 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestFileStorageManager.java
@@ -128,7 +128,7 @@ public class TestFileStorageManager {
}
assertTrue(fs.exists(tablePath));
- FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf, tablePath);
+ FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf);
assertEquals(fs.getUri(), sm.getFileSystem().getUri());
Schema schema = new Schema();
@@ -182,7 +182,7 @@ public class TestFileStorageManager {
DFSTestUtil.createFile(fs, tmpFile, 10, (short) 2, 0xDEADDEADl);
}
assertTrue(fs.exists(tablePath));
- FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf, tablePath);
+ FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(tajoConf);
assertEquals(fs.getUri(), sm.getFileSystem().getUri());
Schema schema = new Schema();
http://git-wip-us.apache.org/repos/asf/tajo/blob/9ad117f0/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index a0daa7d..561e2ef 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@ -95,7 +95,7 @@ public class TestMergeScanner {
conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "parquet", "avro");
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
fs = testDir.getFileSystem(conf);
- sm = StorageManager.getFileStorageManager(conf, testDir);
+ sm = StorageManager.getFileStorageManager(conf);
}
@Test
[2/3] tajo git commit: TAJO-1283: ORDER BY with the first descending
order causes wrong results. (Keuntae Park)
Posted by ji...@apache.org.
TAJO-1283: ORDER BY with the first descending order causes wrong results. (Keuntae Park)
Closes #364
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/02c6c266
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/02c6c266
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/02c6c266
Branch: refs/heads/index_support
Commit: 02c6c266c013d8174a287bc57a6d4131da51ba96
Parents: 9ad117f
Author: Keuntae Park <si...@apache.org>
Authored: Tue Feb 3 16:57:13 2015 +0900
Committer: Keuntae Park <si...@apache.org>
Committed: Tue Feb 3 16:57:13 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../apache/tajo/querymaster/Repartitioner.java | 9 ++----
.../java/org/apache/tajo/querymaster/Stage.java | 3 +-
.../apache/tajo/engine/query/TestSortQuery.java | 34 ++++++++++++++++++++
.../queries/TestSortQuery/testSortFirstDesc.sql | 1 +
.../TestSortQuery/testSortFirstDesc.result | 10 ++++++
.../org/apache/tajo/storage/TupleRange.java | 9 ------
7 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 47eaf9c..5fe46e8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -176,6 +176,9 @@ Release 0.10.0 - unreleased
BUG FIXES
+ TAJO-1283: ORDER BY with the first descending order causes wrong results.
+ (Keuntae Park)
+
TAJO-1324: Remove warehouse directory rewriting in Unit Test. (jinho)
TAJO-1321: Cli prints wrong response time. (jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
index 5a35674..7c2d9f4 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java
@@ -725,13 +725,8 @@ public class Repartitioner {
}
}
- boolean ascendingFirstKey = sortSpecs[0].isAscending();
SortedMap<TupleRange, Collection<FetchImpl>> map;
- if (ascendingFirstKey) {
- map = new TreeMap<TupleRange, Collection<FetchImpl>>();
- } else {
- map = new TreeMap<TupleRange, Collection<FetchImpl>>(new TupleRange.DescendingTupleRangeComparator());
- }
+ map = new TreeMap<TupleRange, Collection<FetchImpl>>();
Set<FetchImpl> fetchSet;
try {
@@ -740,7 +735,7 @@ public class Repartitioner {
fetchSet = new HashSet<FetchImpl>();
for (FetchImpl fetch: fetches) {
String rangeParam =
- TupleUtil.rangeToQuery(ranges[i], ascendingFirstKey ? i == (ranges.length - 1) : i == 0, encoder);
+ TupleUtil.rangeToQuery(ranges[i], i == (ranges.length - 1) , encoder);
FetchImpl copy = null;
try {
copy = fetch.clone();
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 5673d5b..efadaa7 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -1007,7 +1007,8 @@ public class Stage implements EventHandler<StageEvent> {
int mb = (int) Math.ceil((double)volume / 1048576);
LOG.info(stage.getId() + ", Table's volume is approximately " + mb + " MB");
// determine the number of task per 64MB
- int maxTaskNum = Math.max(1, (int) Math.ceil((double)mb / 64));
+ int minTaskNum = Math.max(1, stage.getContext().getQueryMasterContext().getConf().getInt(ConfVars.$TEST_MIN_TASK_NUM.varname, 1));
+ int maxTaskNum = Math.max(minTaskNum, (int) Math.ceil((double)mb / 64));
LOG.info(stage.getId() + ", The determined number of non-leaf tasks is " + maxTaskNum);
return maxTaskNum;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 9c911f9..ff91177 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -102,6 +102,40 @@ public class TestSortQuery extends QueryTestCaseBase {
cleanupQuery(res);
}
+
+ @Test
+ public final void testSortFirstDesc() throws Exception {
+ try {
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2");
+ KeyValueSet tableOptions = new KeyValueSet();
+ tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N");
+
+ Schema schema = new Schema();
+ schema.addColumn("col1", Type.INT4);
+ schema.addColumn("col2", Type.TEXT);
+ String[] data = new String[]{
+ "1|abc",
+ "3|dfa",
+ "3|das",
+ "1|abb",
+ "1|abc",
+ "3|dfb",
+ "3|dat",
+ "1|abe"
+ };
+ TajoTestingCluster.createTable("sortfirstdesc", schema, tableOptions, data, 2);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ } finally {
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0");
+ executeString("DROP TABLE sortfirstdesc PURGE;").close();
+ }
+ }
+
+
@Test
public final void testTopK() throws Exception {
ResultSet res = executeQuery();
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
new file mode 100644
index 0000000..644feb0
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql
@@ -0,0 +1 @@
+select col1, col2 from sortfirstdesc order by col1 desc, col2;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
new file mode 100644
index 0000000..2ed9235
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result
@@ -0,0 +1,10 @@
+col1,col2
+-------------------------------
+3,das
+3,dat
+3,dfa
+3,dfb
+1,abb
+1,abc
+1,abc
+1,abe
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/02c6c266/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
index e824b99..043409a 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java
@@ -94,15 +94,6 @@ public class TupleRange implements Comparable<TupleRange>, Cloneable {
}
}
- public static class DescendingTupleRangeComparator
- implements Comparator<TupleRange> {
-
- @Override
- public int compare(TupleRange left, TupleRange right) {
- return right.compareTo(left);
- }
- }
-
public TupleRange clone() throws CloneNotSupportedException {
TupleRange newRange = (TupleRange) super.clone();
newRange.setStart(start.clone());
[3/3] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1c1dfa0e
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1c1dfa0e
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1c1dfa0e
Branch: refs/heads/index_support
Commit: 1c1dfa0ed2568a561ad6b6f2d251a36d544ee416
Parents: 291817c 02c6c26
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Feb 3 17:01:37 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Feb 3 17:01:37 2015 +0900
----------------------------------------------------------------------
.travis.yml | 2 +-
CHANGES | 5 +++
.../java/org/apache/tajo/master/TajoMaster.java | 9 ++----
.../apache/tajo/querymaster/Repartitioner.java | 9 ++----
.../java/org/apache/tajo/querymaster/Stage.java | 3 +-
.../org/apache/tajo/BackendTestingUtil.java | 5 ++-
.../apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../org/apache/tajo/TajoTestingCluster.java | 12 ++++---
.../planner/physical/TestBNLJoinExec.java | 2 --
.../planner/physical/TestExternalSortExec.java | 2 --
.../physical/TestFullOuterHashJoinExec.java | 2 --
.../physical/TestFullOuterMergeJoinExec.java | 2 --
.../planner/physical/TestHashAntiJoinExec.java | 2 --
.../planner/physical/TestHashJoinExec.java | 2 --
.../planner/physical/TestHashSemiJoinExec.java | 2 --
.../physical/TestLeftOuterHashJoinExec.java | 2 --
.../physical/TestLeftOuterNLJoinExec.java | 2 --
.../planner/physical/TestMergeJoinExec.java | 11 +++----
.../engine/planner/physical/TestNLJoinExec.java | 2 --
.../planner/physical/TestPhysicalPlanner.java | 2 +-
.../physical/TestProgressExternalSortExec.java | 2 --
.../physical/TestRightOuterHashJoinExec.java | 2 --
.../physical/TestRightOuterMergeJoinExec.java | 2 --
.../engine/planner/physical/TestSortExec.java | 2 +-
.../tajo/engine/query/TestCreateTable.java | 2 +-
.../apache/tajo/engine/query/TestSortQuery.java | 34 ++++++++++++++++++++
.../org/apache/tajo/storage/TestRowFile.java | 3 +-
.../queries/TestSortQuery/testSortFirstDesc.sql | 1 +
.../TestSortQuery/testSortFirstDesc.result | 10 ++++++
.../org/apache/tajo/storage/StorageManager.java | 18 +----------
.../org/apache/tajo/storage/TupleRange.java | 9 ------
.../storage/HashShuffleAppenderManager.java | 4 +--
.../tajo/storage/TestFileStorageManager.java | 4 +--
.../apache/tajo/storage/TestMergeScanner.java | 2 +-
34 files changed, 82 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1c1dfa0e/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 2b55745,64efa28..359b483
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@@ -61,9 -61,7 +61,8 @@@ public class TestHashAntiJoinExec
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
private LogicalOptimizer optimizer;
- private StorageManager sm;
private Path testDir;
+ private QueryContext queryContext;
private TableDesc employee;
private TableDesc people;
http://git-wip-us.apache.org/repos/asf/tajo/blob/1c1dfa0e/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index b01cdfd,42a018b..3407972
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@@ -62,9 -62,7 +62,8 @@@ public class TestHashSemiJoinExec
private SQLAnalyzer analyzer;
private LogicalPlanner planner;
private LogicalOptimizer optimizer;
- private StorageManager sm;
private Path testDir;
+ private QueryContext queryContext;
private TableDesc employee;
private TableDesc people;
http://git-wip-us.apache.org/repos/asf/tajo/blob/1c1dfa0e/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/1c1dfa0e/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------