You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/08/14 16:30:28 UTC
[49/51] [partial] tajo git commit: TAJO-1761: Separate an integration
unit test kit into an independent module.
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
new file mode 100644
index 0000000..71ef0ea
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Closeables;
+import com.google.common.io.Files;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.client.TajoClient;
+import org.apache.tajo.client.TajoClientImpl;
+import org.apache.tajo.client.TajoClientUtil;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.engine.planner.global.rewriter.GlobalPlanTestRuleProvider;
+import org.apache.tajo.master.TajoMaster;
+import org.apache.tajo.plan.rewrite.LogicalPlanTestRuleProvider;
+import org.apache.tajo.querymaster.Query;
+import org.apache.tajo.querymaster.QueryMasterTask;
+import org.apache.tajo.querymaster.Stage;
+import org.apache.tajo.querymaster.StageState;
+import org.apache.tajo.service.ServiceTrackerFactory;
+import org.apache.tajo.storage.FileTablespace;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.NetUtils;
+import org.apache.tajo.util.Pair;
+import org.apache.tajo.worker.TajoWorker;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.UUID;
+
+public class TajoTestingCluster {
+ private static Log LOG = LogFactory.getLog(TajoTestingCluster.class);
+ private TajoConf conf;
+ private FileSystem defaultFS;
+ private MiniDFSCluster dfsCluster;
+ private MiniCatalogServer catalogServer;
+ private HBaseTestClusterUtil hbaseUtil;
+
+ private TajoMaster tajoMaster;
+ private List<TajoWorker> tajoWorkers = new ArrayList<TajoWorker>();
+ private boolean isDFSRunning = false;
+ private boolean isTajoClusterRunning = false;
+ private boolean isCatalogServerRunning = false;
+
+ private File clusterTestBuildDir = null;
+
+ /**
+ * Default parent directory for test output.
+ */
+ public static final String DEFAULT_TEST_DIRECTORY = "target/" +
+ System.getProperty("tajo.test.data.dir", "test-data");
+
+ /**
+ * True If HiveCatalogStore is used. Otherwise, it is FALSE.
+ */
+ public Boolean isHiveCatalogStoreUse = false;
+
+ private static final String LOG_LEVEL;
+
+ static {
+ LOG_LEVEL = System.getProperty("LOG_LEVEL");
+ }
+
+ public TajoTestingCluster() {
+ this(false);
+ }
+
+ public TajoTestingCluster(boolean masterHaEMode) {
+ this.conf = new TajoConf();
+ this.conf.setBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE, masterHaEMode);
+
+ initTestDir();
+ setTestingFlagProperties();
+ initPropertiesAndConfigs();
+ }
+
+ void setTestingFlagProperties() {
+ System.setProperty(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
+ conf.set(CommonTestingUtil.TAJO_TEST_KEY, CommonTestingUtil.TAJO_TEST_TRUE);
+ }
+
+ void initPropertiesAndConfigs() {
+
+ // Set time zone
+ TimeZone testDefaultTZ = TimeZone.getTimeZone(TajoConstants.DEFAULT_SYSTEM_TIMEZONE);
+ conf.setSystemTimezone(testDefaultTZ);
+ TimeZone.setDefault(testDefaultTZ);
+
+ // Injection of equality testing code of logical plan (de)serialization
+ conf.setClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, LogicalPlanTestRuleProvider.class);
+ conf.setClassVar(ConfVars.GLOBAL_PLAN_REWRITE_RULE_PROVIDER_CLASS, GlobalPlanTestRuleProvider.class);
+
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES.varname, 4);
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_MEMORY_MB.varname, 2000);
+ conf.setInt(ConfVars.WORKER_RESOURCE_AVAILABLE_DISK_PARALLEL_NUM.varname, 3);
+ conf.setInt(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM.varname, 2);
+
+ // Client API RPC
+ conf.setIntVar(ConfVars.RPC_CLIENT_WORKER_THREAD_NUM, 2);
+
+ //Client API service RPC Server
+ conf.setIntVar(ConfVars.MASTER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.REST_SERVICE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+
+ // Internal RPC Client
+ conf.setIntVar(ConfVars.INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM, 2);
+
+ // Internal RPC Server
+ conf.setIntVar(ConfVars.MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.WORKER_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.CATALOG_RPC_SERVER_WORKER_THREAD_NUM, 2);
+ conf.setIntVar(ConfVars.SHUFFLE_RPC_SERVER_WORKER_THREAD_NUM, 2);
+
+ // Memory cache termination
+ conf.setIntVar(ConfVars.WORKER_HISTORY_EXPIRE_PERIOD, 1);
+
+ // Python function path
+ conf.setStrings(ConfVars.PYTHON_CODE_DIR.varname, getClass().getResource("/python").toString());
+
+ /* Since Travis CI limits the size of standard output log up to 4MB */
+ if (!StringUtils.isEmpty(LOG_LEVEL)) {
+ Level defaultLevel = Logger.getRootLogger().getLevel();
+ Logger.getLogger("org.apache.tajo").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("org.apache.hadoop").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("org.apache.zookeeper").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("BlockStateChange").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ Logger.getLogger("org.mortbay.log").setLevel(Level.toLevel(LOG_LEVEL.toUpperCase(), defaultLevel));
+ }
+ }
+
+ public TajoConf getConfiguration() {
+ return this.conf;
+ }
+
+ public void initTestDir() {
+ if (clusterTestBuildDir == null) {
+ clusterTestBuildDir = setupClusterTestBuildDir();
+ }
+ }
+
+ /**
+ * @return Where to write test data on local filesystem; usually
+ * {@link #DEFAULT_TEST_DIRECTORY}
+ * @see #setupClusterTestBuildDir()
+ */
+ public File getTestDir() {
+ return clusterTestBuildDir;
+ }
+
+ /**
+ * @param subdirName
+ * @return Path to a subdirectory named <code>subdirName</code> under
+ * {@link #getTestDir()}.
+ * @see #setupClusterTestBuildDir()
+ */
+ public static File getTestDir(final String subdirName) {
+ return new File(new File(DEFAULT_TEST_DIRECTORY), subdirName);
+ }
+
+ public static File setupClusterTestBuildDir() {
+ String randomStr = UUID.randomUUID().toString();
+ String dirStr = getTestDir(randomStr).toString();
+ File dir = new File(dirStr).getAbsoluteFile();
+ // Have it cleaned up on exit
+ dir.deleteOnExit();
+ return dir;
+ }
+
+ ////////////////////////////////////////////////////////
+ // HDFS Section
+ ////////////////////////////////////////////////////////
+ /**
+ * Start a minidfscluster.
+ * @param servers How many DNs to start.
+ * @throws Exception
+ * @see {@link #shutdownMiniDFSCluster()}
+ * @return The mini dfs cluster created.
+ */
+ public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception {
+ return startMiniDFSCluster(servers, null, null);
+ }
+
+ /**
+ * Start a minidfscluster.
+ * Can only create one.
+ * @param servers How many DNs to start.
+ * @param dir Where to home your dfs cluster.
+ * @param hosts hostnames DNs to run on.
+ * @throws Exception
+ * @see {@link #shutdownMiniDFSCluster()}
+ * @return The mini dfs cluster created.
+ * @throws java.io.IOException
+ */
+ public MiniDFSCluster startMiniDFSCluster(int servers,
+ File dir,
+ final String hosts[])
+ throws IOException {
+
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString());
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
+ MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf));
+ builder.hosts(hosts);
+ builder.numDataNodes(servers);
+ builder.format(true);
+ builder.manageNameDfsDirs(true);
+ builder.manageDataDfsDirs(true);
+ builder.waitSafeMode(true);
+ this.dfsCluster = builder.build();
+
+ // Set this just-started cluster as our filesystem.
+ this.defaultFS = this.dfsCluster.getFileSystem();
+ this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
+ this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
+ isDFSRunning = true;
+ return this.dfsCluster;
+ }
+
+ public void shutdownMiniDFSCluster() throws Exception {
+ if (this.dfsCluster != null) {
+ try {
+ FileSystem fs = this.dfsCluster.getFileSystem();
+ if (fs != null) fs.close();
+ } catch (IOException e) {
+ System.err.println("error closing file system: " + e);
+ }
+ // The below throws an exception per dn, AsynchronousCloseException.
+ this.dfsCluster.shutdown();
+ }
+ }
+
+ public boolean isRunningDFSCluster() {
+ return this.defaultFS != null;
+ }
+
+ public MiniDFSCluster getMiniDFSCluster() {
+ return this.dfsCluster;
+ }
+
+ public FileSystem getDefaultFileSystem() {
+ return this.defaultFS;
+ }
+
+ public HBaseTestClusterUtil getHBaseUtil() {
+ return hbaseUtil;
+ }
+
+ ////////////////////////////////////////////////////////
+ // Catalog Section
+ ////////////////////////////////////////////////////////
+ public MiniCatalogServer startCatalogCluster() throws Exception {
+ if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running");
+
+ TajoConf c = getConfiguration();
+
+ conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore");
+ conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath() + "/db");
+ LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI));
+ conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0");
+
+ catalogServer = new MiniCatalogServer(conf);
+ CatalogServer catServer = catalogServer.getCatalogServer();
+ InetSocketAddress sockAddr = catServer.getBindAddress();
+ c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr));
+ isCatalogServerRunning = true;
+ return this.catalogServer;
+ }
+
+ public void shutdownCatalogCluster() {
+ if (catalogServer != null) {
+ this.catalogServer.shutdown();
+ }
+ isCatalogServerRunning = false;
+ }
+
+ public MiniCatalogServer getMiniCatalogCluster() {
+ return this.catalogServer;
+ }
+
+ public boolean isHiveCatalogStoreRunning() {
+ return isHiveCatalogStoreUse;
+ }
+
+ ////////////////////////////////////////////////////////
+ // Tajo Cluster Section
+ ////////////////////////////////////////////////////////
+ private void startMiniTajoCluster(File testBuildDir,
+ final int numSlaves,
+ boolean local) throws Exception {
+ TajoConf c = getConfiguration();
+ c.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, "localhost:0");
+ c.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "localhost:0");
+ c.setVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, "localhost:0");
+ c.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, "localhost:0");
+ c.setVar(ConfVars.WORKER_TEMPORAL_DIR, "file://" + testBuildDir.getAbsolutePath() + "/tajo-localdir");
+ c.setIntVar(ConfVars.REST_SERVICE_PORT, 0);
+
+ LOG.info("derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI));
+
+ if (!local) {
+ String tajoRootDir = getMiniDFSCluster().getFileSystem().getUri().toString() + "/tajo";
+ c.setVar(ConfVars.ROOT_DIR, tajoRootDir);
+
+ URI defaultTsUri = TajoConf.getWarehouseDir(c).toUri();
+ FileTablespace defaultTableSpace =
+ new FileTablespace(TablespaceManager.DEFAULT_TABLESPACE_NAME, defaultTsUri);
+ defaultTableSpace.init(conf);
+ TablespaceManager.addTableSpaceForTest(defaultTableSpace);
+
+ } else {
+ c.setVar(ConfVars.ROOT_DIR, "file://" + testBuildDir.getAbsolutePath() + "/tajo");
+ }
+
+ setupCatalogForTesting(c, testBuildDir);
+
+ tajoMaster = new TajoMaster();
+ tajoMaster.init(c);
+ tajoMaster.start();
+
+ this.conf.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, c.getVar(ConfVars.WORKER_PEER_RPC_ADDRESS));
+ this.conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, c.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS));
+
+ InetSocketAddress tajoMasterAddress = tajoMaster.getContext().getTajoMasterService().getBindAddress();
+
+ this.conf.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS,
+ tajoMasterAddress.getHostName() + ":" + tajoMasterAddress.getPort());
+ this.conf.setVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS, c.getVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS));
+ this.conf.setVar(ConfVars.CATALOG_ADDRESS, c.getVar(ConfVars.CATALOG_ADDRESS));
+
+ InetSocketAddress tajoRestAddress = tajoMaster.getContext().getRestServer().getBindAddress();
+
+ this.conf.setIntVar(ConfVars.REST_SERVICE_PORT, tajoRestAddress.getPort());
+
+ startTajoWorkers(numSlaves);
+
+ isTajoClusterRunning = true;
+ LOG.info("Mini Tajo cluster is up");
+ LOG.info("====================================================================================");
+ LOG.info("= MiniTajoCluster starts up =");
+ LOG.info("====================================================================================");
+ LOG.info("= * Master Address: " + tajoMaster.getMasterName());
+ LOG.info("= * CatalogStore: " + tajoMaster.getCatalogServer().getStoreClassName());
+ LOG.info("------------------------------------------------------------------------------------");
+ LOG.info("= * Warehouse Dir: " + TajoConf.getWarehouseDir(c));
+ LOG.info("= * Worker Tmp Dir: " + c.getVar(ConfVars.WORKER_TEMPORAL_DIR));
+ LOG.info("====================================================================================");
+ }
+
+ private void setupCatalogForTesting(TajoConf c, File testBuildDir) throws IOException {
+ final String HIVE_CATALOG_CLASS_NAME = "org.apache.tajo.catalog.store.HiveCatalogStore";
+ boolean hiveCatalogClassExists = false;
+ try {
+ getClass().getClassLoader().loadClass(HIVE_CATALOG_CLASS_NAME);
+ hiveCatalogClassExists = true;
+ } catch (ClassNotFoundException e) {
+ LOG.info("HiveCatalogStore is not available.");
+ }
+ String driverClass = System.getProperty(CatalogConstants.STORE_CLASS);
+
+ if (hiveCatalogClassExists &&
+ driverClass != null && driverClass.equals(HIVE_CATALOG_CLASS_NAME)) {
+ try {
+ getClass().getClassLoader().loadClass(HIVE_CATALOG_CLASS_NAME);
+ String jdbcUri = "jdbc:derby:;databaseName="+ testBuildDir.toURI().getPath() + "/metastore_db;create=true";
+ c.set("hive.metastore.warehouse.dir", TajoConf.getWarehouseDir(c).toString() + "/default");
+ c.set("javax.jdo.option.ConnectionURL", jdbcUri);
+ c.set(TajoConf.ConfVars.WAREHOUSE_DIR.varname, conf.getVar(ConfVars.WAREHOUSE_DIR));
+ c.set(CatalogConstants.STORE_CLASS, HIVE_CATALOG_CLASS_NAME);
+ Path defaultDatabasePath = new Path(TajoConf.getWarehouseDir(c).toString() + "/default");
+ FileSystem fs = defaultDatabasePath.getFileSystem(c);
+ if (!fs.exists(defaultDatabasePath)) {
+ fs.mkdirs(defaultDatabasePath);
+ }
+ isHiveCatalogStoreUse = true;
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException(cnfe);
+ }
+ } else { // for derby
+ c.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore");
+ c.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + testBuildDir.getAbsolutePath() + "/db");
+ }
+ c.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0");
+ }
+
+ private void startTajoWorkers(int numSlaves) throws Exception {
+ for(int i = 0; i < 1; i++) {
+ TajoWorker tajoWorker = new TajoWorker();
+
+ TajoConf workerConf = new TajoConf(this.conf);
+
+ workerConf.setVar(ConfVars.WORKER_INFO_ADDRESS, "localhost:0");
+ workerConf.setVar(ConfVars.WORKER_CLIENT_RPC_ADDRESS, "localhost:0");
+ workerConf.setVar(ConfVars.WORKER_PEER_RPC_ADDRESS, "localhost:0");
+
+ workerConf.setVar(ConfVars.WORKER_QM_RPC_ADDRESS, "localhost:0");
+
+ tajoWorker.startWorker(workerConf, new String[0]);
+
+ LOG.info("MiniTajoCluster Worker #" + (i + 1) + " started.");
+ tajoWorkers.add(tajoWorker);
+ }
+ }
+
+ public TajoMaster getMaster() {
+ return this.tajoMaster;
+ }
+
+ public List<TajoWorker> getTajoWorkers() {
+ return this.tajoWorkers;
+ }
+
+ public void shutdownMiniTajoCluster() {
+ if(this.tajoMaster != null) {
+ this.tajoMaster.stop();
+ }
+ for(TajoWorker eachWorker: tajoWorkers) {
+ eachWorker.stopWorkerForce();
+ }
+ tajoWorkers.clear();
+ this.tajoMaster= null;
+ }
+
+ ////////////////////////////////////////////////////////
+ // Meta Cluster Section
+ ////////////////////////////////////////////////////////
+ /**
+ * @throws java.io.IOException If a cluster -- dfs or engine -- already running.
+ */
+ void isRunningCluster() throws IOException {
+ if (!isTajoClusterRunning && !isCatalogServerRunning && !isDFSRunning) return;
+ throw new IOException("Cluster already running at " +
+ this.clusterTestBuildDir);
+ }
+
+ /**
+ * This method starts up a tajo cluster with a given number of clusters in
+ * distributed mode.
+ *
+ * @param numSlaves the number of tajo cluster to start up
+ * @throws Exception
+ */
+ public void startMiniCluster(final int numSlaves)
+ throws Exception {
+ startMiniCluster(numSlaves, null);
+ }
+
+ public void startMiniCluster(final int numSlaves, final String [] dataNodeHosts) throws Exception {
+
+ int numDataNodes = numSlaves;
+ if(dataNodeHosts != null && dataNodeHosts.length != 0) {
+ numDataNodes = dataNodeHosts.length;
+ }
+
+ LOG.info("Starting up minicluster with 1 master(s) and " +
+ numSlaves + " worker(s) and " + numDataNodes + " datanode(s)");
+
+ // If we already bring up the cluster, fail.
+ isRunningCluster();
+ if (clusterTestBuildDir != null) {
+ LOG.info("Using passed path: " + clusterTestBuildDir);
+ }
+
+ startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts);
+ this.dfsCluster.waitClusterUp();
+
+ conf.setInt("hbase.hconnection.threads.core", 5);
+ conf.setInt("hbase.hconnection.threads.max", 50);
+ hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir);
+
+ startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, false);
+ }
+
+ public void startMiniClusterInLocal(final int numSlaves) throws Exception {
+ isRunningCluster();
+
+ if (clusterTestBuildDir != null) {
+ LOG.info("Using passed path: " + clusterTestBuildDir);
+ }
+
+ startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, true);
+ }
+
+ public void shutdownMiniCluster() throws IOException {
+ LOG.info("========================================");
+ LOG.info("Minicluster is stopping");
+ LOG.info("========================================");
+
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ shutdownMiniTajoCluster();
+
+ if(this.catalogServer != null) {
+ shutdownCatalogCluster();
+ isCatalogServerRunning = false;
+ }
+
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ if(this.dfsCluster != null) {
+ try {
+ FileSystem fs = this.dfsCluster.getFileSystem();
+ if (fs != null) fs.close();
+ this.dfsCluster.shutdown();
+ } catch (IOException e) {
+ System.err.println("error closing file system: " + e);
+ }
+ isDFSRunning = false;
+ }
+
+ if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
+ if(!ShutdownHookManager.get().isShutdownInProgress()) {
+ //TODO clean test dir when ShutdownInProgress
+ LocalFileSystem localFS = LocalFileSystem.getLocal(conf);
+ localFS.delete(new Path(clusterTestBuildDir.toString()), true);
+ localFS.close();
+ }
+ this.clusterTestBuildDir = null;
+ }
+
+ if(hbaseUtil != null) {
+ hbaseUtil.stopZooKeeperCluster();
+ hbaseUtil.stopHBaseCluster();
+ }
+
+ LOG.info("Minicluster is down");
+ isTajoClusterRunning = false;
+ }
+
+ public TajoClient newTajoClient() throws Exception {
+ return new TajoClientImpl(ServiceTrackerFactory.get(getConfiguration()));
+ }
+
+ public static ResultSet run(String[] names,
+ Schema[] schemas,
+ KeyValueSet tableOption,
+ String[][] tables,
+ String query,
+ TajoClient client) throws Exception {
+ TajoTestingCluster util = TpchTestBase.getInstance().getTestingCluster();
+
+ FileSystem fs = util.getDefaultFileSystem();
+ Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration());
+ fs.mkdirs(rootDir);
+ for (int i = 0; i < names.length; i++) {
+ createTable(names[i], schemas[i], tableOption, tables[i]);
+ }
+ Thread.sleep(1000);
+ ResultSet res = client.executeQueryAndGetResult(query);
+ return res;
+ }
+
+ public static ResultSet run(String[] names,
+ Schema[] schemas,
+ KeyValueSet tableOption,
+ String[][] tables,
+ String query) throws Exception {
+ TpchTestBase instance = TpchTestBase.getInstance();
+ TajoTestingCluster util = instance.getTestingCluster();
+ while(true) {
+ if(util.getMaster().isMasterRunning()) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ TajoConf conf = util.getConfiguration();
+ TajoClient client = new TajoClientImpl(ServiceTrackerFactory.get(conf));
+
+ try {
+ return run(names, schemas, tableOption, tables, query, client);
+ } finally {
+ client.close();
+ }
+ }
+
+ public static TajoClient newTajoClient(TajoTestingCluster util) throws SQLException, InterruptedException {
+ while(true) {
+ if(util.getMaster().isMasterRunning()) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ TajoConf conf = util.getConfiguration();
+ return new TajoClientImpl(ServiceTrackerFactory.get(conf));
+ }
+
+ public static void createTable(String tableName, Schema schema,
+ KeyValueSet tableOption, String[] tableDatas) throws Exception {
+ createTable(tableName, schema, tableOption, tableDatas, 1);
+ }
+
+ public static void createTable(String tableName, Schema schema,
+ KeyValueSet tableOption, String[] tableDatas, int numDataFiles) throws Exception {
+ TpchTestBase instance = TpchTestBase.getInstance();
+ TajoTestingCluster util = instance.getTestingCluster();
+ TajoClient client = newTajoClient(util);
+ try {
+ FileSystem fs = util.getDefaultFileSystem();
+ Path rootDir = TajoConf.getWarehouseDir(util.getConfiguration());
+ if (!fs.exists(rootDir)) {
+ fs.mkdirs(rootDir);
+ }
+ Path tablePath;
+ if (CatalogUtil.isFQTableName(tableName)) {
+ Pair<String, String> name = CatalogUtil.separateQualifierAndName(tableName);
+ tablePath = new Path(rootDir, new Path(name.getFirst(), name.getSecond()));
+ } else {
+ tablePath = new Path(rootDir, tableName);
+ }
+
+ fs.mkdirs(tablePath);
+ if (tableDatas.length > 0) {
+ int recordPerFile = tableDatas.length / numDataFiles;
+ if (recordPerFile == 0) {
+ recordPerFile = 1;
+ }
+ FSDataOutputStream out = null;
+ for (int j = 0; j < tableDatas.length; j++) {
+ if (out == null || j % recordPerFile == 0) {
+ if (out != null) {
+ out.close();
+ }
+ Path dfsPath = new Path(tablePath, tableName + j + ".tbl");
+ out = fs.create(dfsPath);
+ }
+ out.write((tableDatas[j] + "\n").getBytes());
+ }
+ if (out != null) {
+ out.close();
+ }
+ }
+ TableMeta meta = CatalogUtil.newTableMeta("TEXT", tableOption);
+ client.createExternalTable(tableName, schema, tablePath.toUri(), meta);
+ } finally {
+ client.close();
+ }
+ }
+
+ /**
+ * Write lines to a file.
+ *
+ * @param file File to write lines to
+ * @param lines Strings written to the file
+ * @throws java.io.IOException
+ */
+ private static void writeLines(File file, String... lines)
+ throws IOException {
+ Writer writer = Files.newWriter(file, Charsets.UTF_8);
+ try {
+ for (String line : lines) {
+ writer.write(line);
+ writer.write('\n');
+ }
+ } finally {
+ Closeables.closeQuietly(writer);
+ }
+ }
+
+ public void setAllTajoDaemonConfValue(String key, String value) {
+ tajoMaster.getContext().getConf().set(key, value);
+ setAllWorkersConfValue(key, value);
+ }
+
+ public void setAllWorkersConfValue(String key, String value) {
+ for (TajoWorker eachWorker: tajoWorkers) {
+ eachWorker.getConfig().set(key, value);
+ }
+ }
+
+ public void waitForQuerySubmitted(QueryId queryId) throws Exception {
+ waitForQuerySubmitted(queryId, 50);
+ }
+
+ public void waitForQuerySubmitted(QueryId queryId, int delay) throws Exception {
+ QueryMasterTask qmt = null;
+
+ int i = 0;
+ while (qmt == null || TajoClientUtil.isQueryWaitingForSchedule(qmt.getState())) {
+ try {
+ Thread.sleep(delay);
+
+ if (qmt == null) {
+ qmt = getQueryMasterTask(queryId);
+ }
+ } catch (InterruptedException e) {
+ }
+ if (++i > 200) {
+ throw new IOException("Timed out waiting for query to start");
+ }
+ }
+ }
+
+ public void waitForQueryState(Query query, TajoProtos.QueryState expected, int delay) throws Exception {
+ int i = 0;
+ while (query == null || query.getSynchronizedState() != expected) {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+ }
+ if (++i > 200) {
+ throw new IOException("Timed out waiting. expected: " + expected +
+ ", actual: " + query != null ? String.valueOf(query.getSynchronizedState()) : String.valueOf(query));
+ }
+ }
+ }
+
+ public void waitForStageState(Stage stage, StageState expected, int delay) throws Exception {
+
+ int i = 0;
+ while (stage == null || stage.getSynchronizedState() != expected) {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+ }
+ if (++i > 200) {
+ throw new IOException("Timed out waiting");
+ }
+ }
+ }
+
+ public QueryMasterTask getQueryMasterTask(QueryId queryId) {
+ QueryMasterTask qmt = null;
+ for (TajoWorker worker : getTajoWorkers()) {
+ qmt = worker.getWorkerContext().getQueryMaster().getQueryMasterTask(queryId, true);
+ if (qmt != null && queryId.equals(qmt.getQueryId())) {
+ break;
+ }
+ }
+ return qmt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
new file mode 100644
index 0000000..055dd02
--- /dev/null
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TpchTestBase.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.benchmark.TPCH;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.FileUtil;
+import org.apache.tajo.util.KeyValueSet;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.util.Map;
+
+public class TpchTestBase {
+ private static final Log LOG = LogFactory.getLog(TpchTestBase.class);
+
+ String [] names;
+ String [] paths;
+ String [][] tables;
+ Schema[] schemas;
+ Map<String, Integer> nameMap = Maps.newHashMap();
+ protected TPCH tpch;
+ protected LocalTajoTestingUtility util;
+
+ private static TpchTestBase testBase;
+
+ static {
+ try {
+ testBase = new TpchTestBase();
+ testBase.setUp();
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ private TpchTestBase() throws IOException {
+ names = new String[] {"customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier", "empty_orders"};
+ paths = new String[names.length];
+ for (int i = 0; i < names.length; i++) {
+ nameMap.put(names[i], i);
+ }
+
+ tpch = new TPCH();
+ tpch.loadSchemas();
+ tpch.loadQueries();
+
+ schemas = new Schema[names.length];
+ for (int i = 0; i < names.length; i++) {
+ schemas[i] = tpch.getSchema(names[i]);
+ }
+
+ tables = new String[names.length][];
+ File file;
+ for (int i = 0; i < names.length; i++) {
+ file = TPCH.getDataFile(names[i]);
+ tables[i] = FileUtil.readTextFile(file).split("\n");
+ paths[i] = file.getAbsolutePath();
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void setUp() throws Exception {
+ util = new LocalTajoTestingUtility();
+ KeyValueSet opt = new KeyValueSet();
+ opt.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ util.setup(names, paths, schemas, opt);
+ }
+
+ public static TpchTestBase getInstance() {
+ return testBase;
+ }
+
+ public ResultSet execute(String query) throws Exception {
+ return util.execute(query);
+ }
+
+ public TajoTestingCluster getTestingCluster() {
+ return util.getTestingCluster();
+ }
+
+ public void tearDown() throws IOException {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ }
+ util.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml
new file mode 100644
index 0000000..8f2051d
--- /dev/null
+++ b/tajo-core-tests/pom.xml
@@ -0,0 +1,356 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>tajo-project</artifactId>
+ <groupId>org.apache.tajo</groupId>
+ <version>0.11.0-SNAPSHOT</version>
+ <relativePath>../tajo-project</relativePath>
+ </parent>
+ <artifactId>tajo-core-tests</artifactId>
+ <packaging>jar</packaging>
+ <name>Tajo Core Tests</name>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>derby.log</exclude>
+ <exclude>benchmark/**</exclude>
+ <exclude>src/test/tpch/**</exclude>
+ <exclude>src/test/resources/dataset/**</exclude>
+ <exclude>src/test/resources/queries/**</exclude>
+ <exclude>src/test/resources/results/**</exclude>
+ <exclude>src/main/resources/META-INF/services/*</exclude>
+ <exclude>src/main/resources/webapps/static/js/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <tajo.test>TRUE</tajo.test>
+ </systemProperties>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.3.1</version>
+ <configuration>
+ <excludes>
+ <exclude>LICENSE</exclude>
+ </excludes>
+ <archive>
+ <addMavenDescriptor>false</addMavenDescriptor>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeScope>runtime</includeScope>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-algebra</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-catalog-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-plan</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-catalog-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-catalog-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-storage-hbase</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-pullserver</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-rpc-protobuf</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-thirdparty-asm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-ws-rs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-metrics</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-cluster-tests</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>jdk.tools</artifactId>
+ <groupId>jdk.tools</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.stephenc.jcip</groupId>
+ <artifactId>jcip-annotations</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>parallel-test</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration combine.self="override">
+ <forkCount>${maven.fork.count}</forkCount>
+ <reuseForks>true</reuseForks>
+ <trimStackTrace>false</trimStackTrace>
+ <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=152m -Dfile.encoding=UTF-8</argLine>
+ <useSystemClassLoader>true</useSystemClassLoader>
+ <useManifestOnlyJar>true</useManifestOnlyJar>
+ <systemProperties>
+ <tajo.test>TRUE</tajo.test>
+ <tajo.test.data.dir>test-data${surefire.forkNumber}</tajo.test.data.dir>
+ </systemProperties>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/log4j.properties
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/log4j.properties b/tajo-core-tests/src/test/java/log4j.properties
new file mode 100644
index 0000000..48f9d8e
--- /dev/null
+++ b/tajo-core-tests/src/test/java/log4j.properties
@@ -0,0 +1,28 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p: %c (%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.hadoop.conf=ERROR
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java b/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java
new file mode 100644
index 0000000..8dc95de
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/TestQueryIdFactory.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import org.apache.tajo.engine.planner.global.MasterPlan;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class TestQueryIdFactory {
+
+ @Before
+ public void setup() {
+ }
+
+ @Test
+ public void testNewQueryId() {
+ QueryId qid1 = LocalTajoTestingUtility.newQueryId();
+ QueryId qid2 = LocalTajoTestingUtility.newQueryId();
+ assertTrue(qid1.compareTo(qid2) < 0);
+ }
+
+ @Test
+ public void testNewStageId() {
+ QueryId qid = LocalTajoTestingUtility.newQueryId();
+ MasterPlan plan = new MasterPlan(qid, null, null);
+ ExecutionBlockId stageId1 = plan.newExecutionBlockId();
+ ExecutionBlockId stageId2 = plan.newExecutionBlockId();
+ assertTrue(stageId1.compareTo(stageId2) < 0);
+ }
+
+ @Test
+ public void testNewTaskId() {
+ QueryId qid = LocalTajoTestingUtility.newQueryId();
+ MasterPlan plan = new MasterPlan(qid, null, null);
+ ExecutionBlockId subid = plan.newExecutionBlockId();
+ TaskId quid1 = QueryIdFactory.newTaskId(subid);
+ TaskId quid2 = QueryIdFactory.newTaskId(subid);
+ assertTrue(quid1.compareTo(quid2) < 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java b/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java
new file mode 100644
index 0000000..d15e282
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/TestTajoIds.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.tajo.engine.planner.global.MasterPlan;
+import org.apache.tajo.util.TajoIdUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestTajoIds {
+ @Test
+ public void testQueryId() {
+ long ts1 = 1315890136000l;
+ long ts2 = 1315890136001l;
+
+ QueryId j1 = createQueryId(ts1, 2);
+ QueryId j2 = createQueryId(ts1, 1);
+ QueryId j3 = createQueryId(ts2, 1);
+ QueryId j4 = createQueryId(ts1, 2);
+
+ assertTrue(j1.equals(j4));
+ assertFalse(j1.equals(j2));
+ assertFalse(j1.equals(j3));
+
+ assertTrue(j1.compareTo(j4) == 0);
+ assertTrue(j1.compareTo(j2) > 0);
+ assertTrue(j1.compareTo(j3) < 0);
+
+ assertTrue(j1.hashCode() == j4.hashCode());
+ assertFalse(j1.hashCode() == j2.hashCode());
+ assertFalse(j1.hashCode() == j3.hashCode());
+
+ QueryId j5 = createQueryId(ts1, 231415);
+ assertEquals("q_" + ts1 + "_0002", j1.toString());
+ assertEquals("q_" + ts1 + "_231415", j5.toString());
+ }
+
+ @Test
+ public void testQueryIds() {
+ long timeId = 1315890136000l;
+
+ QueryId queryId = createQueryId(timeId, 1);
+ assertEquals("q_" + timeId + "_0001", queryId.toString());
+
+ ExecutionBlockId subId = QueryIdFactory.newExecutionBlockId(queryId, 2);
+ assertEquals("eb_" + timeId +"_0001_000002", subId.toString());
+
+ TaskId qId = new TaskId(subId, 5);
+ assertEquals("t_" + timeId + "_0001_000002_000005", qId.toString());
+
+ TaskAttemptId attemptId = new TaskAttemptId(qId, 4);
+ assertEquals("ta_" + timeId + "_0001_000002_000005_04", attemptId.toString());
+ }
+
+ @Test
+ public void testEqualsObject() {
+ long timeId = System.currentTimeMillis();
+
+ QueryId queryId1 = createQueryId(timeId, 1);
+ QueryId queryId2 = createQueryId(timeId, 2);
+ assertNotSame(queryId1, queryId2);
+ QueryId queryId3 = createQueryId(timeId, 1);
+ assertEquals(queryId1, queryId3);
+
+ ExecutionBlockId sid1 = QueryIdFactory.newExecutionBlockId(queryId1, 1);
+ ExecutionBlockId sid2 = QueryIdFactory.newExecutionBlockId(queryId1, 2);
+ assertNotSame(sid1, sid2);
+ ExecutionBlockId sid3 = QueryIdFactory.newExecutionBlockId(queryId1, 1);
+ assertEquals(sid1, sid3);
+
+ TaskId qid1 = new TaskId(sid1, 9);
+ TaskId qid2 = new TaskId(sid1, 10);
+ assertNotSame(qid1, qid2);
+ TaskId qid3 = new TaskId(sid1, 9);
+ assertEquals(qid1, qid3);
+ }
+
+ @Test
+ public void testCompareTo() {
+ long time = System.currentTimeMillis();
+
+ QueryId queryId1 = createQueryId(time, 1);
+ QueryId queryId2 = createQueryId(time, 2);
+ QueryId queryId3 = createQueryId(time, 1);
+ assertEquals(-1, queryId1.compareTo(queryId2));
+ assertEquals(1, queryId2.compareTo(queryId1));
+ assertEquals(0, queryId3.compareTo(queryId1));
+
+ ExecutionBlockId sid1 = QueryIdFactory.newExecutionBlockId(queryId1, 1);
+ ExecutionBlockId sid2 = QueryIdFactory.newExecutionBlockId(queryId1, 2);
+ ExecutionBlockId sid3 = QueryIdFactory.newExecutionBlockId(queryId1, 1);
+ assertEquals(-1, sid1.compareTo(sid2));
+ assertEquals(1, sid2.compareTo(sid1));
+ assertEquals(0, sid3.compareTo(sid1));
+
+ TaskId qid1 = new TaskId(sid1, 9);
+ TaskId qid2 = new TaskId(sid1, 10);
+ TaskId qid3 = new TaskId(sid1, 9);
+ assertEquals(-1, qid1.compareTo(qid2));
+ assertEquals(1, qid2.compareTo(qid1));
+ assertEquals(0, qid3.compareTo(qid1));
+ }
+
+ @Test
+ public void testConstructFromString() {
+ QueryId qid1 = LocalTajoTestingUtility.newQueryId();
+ QueryId qid2 = TajoIdUtils.parseQueryId(qid1.toString());
+ assertEquals(qid1, qid2);
+
+ MasterPlan plan1 = new MasterPlan(qid1, null, null);
+ ExecutionBlockId sub1 = plan1.newExecutionBlockId();
+ ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
+ assertEquals(sub1, sub2);
+
+ TaskId u1 = QueryIdFactory.newTaskId(sub1);
+ TaskId u2 = new TaskId(u1.getProto());
+ assertEquals(u1, u2);
+
+ TaskAttemptId attempt1 = new TaskAttemptId(u1, 1);
+ TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto());
+ assertEquals(attempt1, attempt2);
+ }
+
+ @Test
+ public void testConstructFromPB() {
+ QueryId qid1 = LocalTajoTestingUtility.newQueryId();
+ QueryId qid2 = new QueryId(qid1.getProto());
+ assertEquals(qid1, qid2);
+
+ MasterPlan plan = new MasterPlan(qid1, null, null);
+ ExecutionBlockId sub1 = plan.newExecutionBlockId();
+ ExecutionBlockId sub2 = TajoIdUtils.createExecutionBlockId(sub1.toString());
+ assertEquals(sub1, sub2);
+
+ TaskId u1 = QueryIdFactory.newTaskId(sub1);
+ TaskId u2 = new TaskId(u1.getProto());
+ assertEquals(u1, u2);
+
+ TaskAttemptId attempt1 = new TaskAttemptId(u1, 1);
+ TaskAttemptId attempt2 = new TaskAttemptId(attempt1.getProto());
+ assertEquals(attempt1, attempt2);
+ }
+
+ public static QueryId createQueryId(long timestamp, int id) {
+ ApplicationId appId = BuilderUtils.newApplicationId(timestamp, id);
+
+ return QueryIdFactory.newQueryId(appId.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java b/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
new file mode 100644
index 0000000..53d4350
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.benchmark;
+
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.TajoConstants;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(IntegrationTest.class)
+public class TestTPCH extends QueryTestCaseBase {
+
+ public TestTPCH() {
+ super(TajoConstants.DEFAULT_DATABASE_NAME);
+ }
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testQ1OrderBy() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testQ2FourJoins() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testTPCH14Expr() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testTPCHQ5() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testFirstJoinInQ7() throws Exception {
+ runSimpleTests();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
new file mode 100644
index 0000000..06a54c4
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.cli.tools;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.util.FileUtil;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+
+public class TestDDLBuilder {
+ private static final Schema schema1;
+ private static final TableMeta meta1;
+ private static final PartitionMethodDesc partitionMethod1;
+
+ static {
+ schema1 = new Schema();
+ schema1.addColumn("name", TajoDataTypes.Type.BLOB);
+ schema1.addColumn("addr", TajoDataTypes.Type.TEXT);
+
+ meta1 = CatalogUtil.newTableMeta("TEXT");
+ meta1.putOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ meta1.putOption(StorageConstants.COMPRESSION_CODEC, GzipCodec.class.getName());
+
+ Schema expressionSchema = new Schema();
+ expressionSchema.addColumn("key", TajoDataTypes.Type.INT4);
+ expressionSchema.addColumn("key2", TajoDataTypes.Type.TEXT);
+ partitionMethod1 = new PartitionMethodDesc(
+ "db1",
+ "table1",
+ CatalogProtos.PartitionType.COLUMN,
+ "key,key2",
+ expressionSchema);
+ }
+
+ @Test
+ public void testBuildDDLForExternalTable() throws Exception {
+ TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri());
+ desc.setPartitionMethod(partitionMethod1);
+ desc.setExternal(true);
+ assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
+ DDLBuilder.buildDDLForExternalTable(desc));
+ }
+
+ @Test
+ public void testBuildDDLQuotedTableName() throws Exception {
+ Schema schema2 = new Schema();
+ schema2.addColumn("name", TajoDataTypes.Type.BLOB);
+ schema2.addColumn("addr", TajoDataTypes.Type.TEXT);
+ schema2.addColumn("FirstName", TajoDataTypes.Type.TEXT);
+ schema2.addColumn("LastName", TajoDataTypes.Type.TEXT);
+ schema2.addColumn("with", TajoDataTypes.Type.TEXT);
+
+ Schema expressionSchema2 = new Schema();
+ expressionSchema2.addColumn("BirthYear", TajoDataTypes.Type.INT4);
+
+ PartitionMethodDesc partitionMethod2 = new PartitionMethodDesc(
+ "db1",
+ "table1",
+ CatalogProtos.PartitionType.COLUMN,
+ "key,key2",
+ expressionSchema2);
+
+ TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri());
+ desc.setPartitionMethod(partitionMethod2);
+ desc.setExternal(true);
+ assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
+ DDLBuilder.buildDDLForExternalTable(desc));
+
+ desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri());
+ desc.setPartitionMethod(partitionMethod2);
+ desc.setExternal(false);
+ assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
+ DDLBuilder.buildDDLForBaseTable(desc));
+ }
+
+ @Test
+ public void testBuildDDLForBaseTable() throws Exception {
+ TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri());
+ assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
+ DDLBuilder.buildDDLForBaseTable(desc));
+ }
+
+ @Test
+ public void testBuildColumn() throws Exception {
+ String [] tobeUnquoted = {
+ "column_name",
+ "columnname",
+ "column_1",
+ };
+
+ for (String columnName : tobeUnquoted) {
+ assertFalse(CatalogUtil.isShouldBeQuoted(columnName));
+ }
+
+ String [] quoted = {
+ "Column_Name",
+ "COLUMN_NAME",
+ "컬럼",
+ "$column_name",
+ "Column_Name1",
+ "with",
+ "when"
+ };
+
+ for (String columnName : quoted) {
+ assertTrue(CatalogUtil.isShouldBeQuoted(columnName));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
new file mode 100644
index 0000000..aa8070e
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.cli.tools;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.auth.UserRoleInfo;
+import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.storage.TablespaceManager;
+import org.apache.tajo.util.FileUtil;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintWriter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestTajoDump extends QueryTestCaseBase {
+
+ @Test
+ public void testDump1() throws Exception {
+ if (!testingCluster.isHiveCatalogStoreRunning()) {
+ executeString("CREATE TABLE \"" + getCurrentDatabase() +
+ "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)");
+
+ try {
+ UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(bos);
+ TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
+ printWriter.flush();
+ printWriter.close();
+ assertStrings(new String(bos.toByteArray()));
+ bos.close();
+ } finally {
+ executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\"");
+ }
+ }
+ }
+
+ @Test
+ public void testDump2() throws Exception {
+ if (!testingCluster.isHiveCatalogStoreRunning()) {
+ executeString("CREATE TABLE \"" + getCurrentDatabase() +
+ "\".\"TableName2\" (\"Age\" int, \"Name\" Record (\"FirstName\" TEXT, lastname TEXT))");
+
+ try {
+ UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(bos);
+ TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
+ printWriter.flush();
+ printWriter.close();
+ assertStrings(new String(bos.toByteArray()));
+ bos.close();
+ } finally {
+ executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\"");
+ }
+ }
+ }
+
+ @Test
+ public void testDump3() throws Exception {
+ if (!testingCluster.isHiveCatalogStoreRunning()) {
+ executeString("CREATE TABLE \"" + getCurrentDatabase() +
+ "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)");
+
+ executeString("CREATE INDEX test_idx on \"" + getCurrentDatabase()
+ + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )");
+
+ try {
+ UserRoleInfo userInfo = UserRoleInfo.getCurrentUser();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter printWriter = new PrintWriter(bos);
+ TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter);
+ printWriter.flush();
+ printWriter.close();
+ assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"},
+ new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()});
+ bos.close();
+ } finally {
+ executeString("DROP INDEX test_idx");
+ executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\"");
+ }
+ }
+ }
+
+ private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues)
+ throws Exception {
+ FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
+ Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile);
+ assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile));
+
+ String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
+
+ if (paramKeys != null) {
+ for (int i = 0; i < paramKeys.length; i++) {
+ if (i < paramValues.length) {
+ expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]);
+ }
+ }
+ }
+ assertEquals(expectedResult.trim(), actual.trim());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a4106883/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
new file mode 100644
index 0000000..3b53c60
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.cli.tsql;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.TajoTestingCluster;
+import org.apache.tajo.TpchTestBase;
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.Float8Datum;
+import org.apache.tajo.datum.Int4Datum;
+import org.apache.tajo.datum.TextDatum;
+import org.apache.tajo.jdbc.MetaDataTuple;
+import org.apache.tajo.jdbc.TajoMetaDataResultSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.net.URL;
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestDefaultCliOutputFormatter {
+ protected static final TpchTestBase testBase;
+ protected static final TajoTestingCluster cluster;
+
+ /** the base path of result directories */
+ protected static final Path resultBasePath;
+ static {
+ testBase = TpchTestBase.getInstance();
+ cluster = testBase.getTestingCluster();
+ URL resultBaseURL = ClassLoader.getSystemResource("results");
+ resultBasePath = new Path(resultBaseURL.toString());
+ }
+
+ private TajoConf conf;
+ private TajoCli tajoCli;
+ private TajoCli.TajoCliContext cliContext;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = cluster.getConfiguration();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ tajoCli = new TajoCli(conf, new String[]{}, System.in, out);
+ cliContext = tajoCli.getContext();
+ }
+
+ @After
+ public void tearDown() {
+ if (tajoCli != null) {
+ tajoCli.close();
+ }
+ }
+
+
+ @Test
+ public void testParseErrorMessage() {
+ String message = "java.sql.SQLException: ERROR: no such a table: table1";
+ assertEquals("ERROR: no such a table: table1", DefaultTajoCliOutputFormatter.parseErrorMessage(message));
+
+ String multiLineMessage =
+ "ERROR: java.sql.SQLException: ERROR: no such a table: table1\n" +
+ "com.google.protobuf.ServiceException: java.sql.SQLException: ERROR: no such a table: table1\n" +
+ "\tat org.apache.tajo.client.TajoClient.getTableDesc(TajoClient.java:777)\n" +
+ "\tat org.apache.tajo.cli.tsql.commands.DescTableCommand.invoke(DescTableCommand.java:43)\n" +
+ "\tat org.apache.tajo.cli.tsql.TajoCli.executeMetaCommand(TajoCli.java:300)\n" +
+ "\tat org.apache.tajo.cli.tsql.TajoCli.executeParsedResults(TajoCli.java:280)\n" +
+ "\tat org.apache.tajo.cli.tsql.TajoCli.runShell(TajoCli.java:271)\n" +
+ "\tat org.apache.tajo.cli.tsql.TajoCli.main(TajoCli.java:420)\n" +
+ "Caused by: java.sql.SQLException: ERROR: no such a table: table1\n" +
+ "\t... 6 more";
+
+ assertEquals(multiLineMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(multiLineMessage));
+
+ String noPrefixMessage = "RTFM please";
+ assertEquals("ERROR: "+noPrefixMessage, DefaultTajoCliOutputFormatter.parseErrorMessage(noPrefixMessage));
+
+ String errorMessageWithLine = "ERROR: syntax error at or near '('\n" +
+ "LINE 1:7 select (*) from tc\n" +
+ " ^";
+ assertEquals(errorMessageWithLine, DefaultTajoCliOutputFormatter.parseErrorMessage(errorMessageWithLine));
+ }
+
+ @Test
+ public void testPrintResultInsertStatement() throws Exception {
+ DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter();
+ outputFormatter.init(cliContext);
+
+ float responseTime = 10.1f;
+ long numBytes = 102;
+ long numRows = 30;
+
+ TableDesc tableDesc = new TableDesc();
+ TableStats stats = new TableStats();
+ stats.setNumBytes(102);
+ stats.setNumRows(numRows);
+ tableDesc.setStats(stats);
+
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ outputFormatter.printResult(writer, null, tableDesc, responseTime, null);
+
+ String expectedOutput = "(" + numRows + " rows, " + responseTime + " sec, " + numBytes + " B inserted)\n";
+ assertEquals(expectedOutput, stringWriter.toString());
+ }
+
+ @Test
+ public void testPrintResultSelectStatement() throws Exception {
+ DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter();
+ outputFormatter.init(cliContext);
+
+ float responseTime = 10.1f;
+ long numBytes = 102;
+ long numRows = 30;
+
+ TableDesc tableDesc = new TableDesc();
+ TableStats stats = new TableStats();
+ stats.setNumBytes(102);
+ stats.setNumRows(numRows);
+ tableDesc.setStats(stats);
+
+ final List<MetaDataTuple> resultTables = new ArrayList<MetaDataTuple>();
+
+ String expectedOutput = "col1, col2, col3\n";
+ expectedOutput += "-------------------------------\n";
+
+ String prefix = "";
+ for (int i = 0; i < numRows; i++) {
+ MetaDataTuple tuple = new MetaDataTuple(3);
+
+ int index = 0;
+
+ tuple.put(index++, new TextDatum("row_" + i));
+ tuple.put(index++, new Int4Datum(i));
+ tuple.put(index++, new Float8Datum(i));
+
+ expectedOutput += prefix + "row_" + i + ", " + (new Int4Datum(i)) + ", " + (new Float8Datum(i));
+ prefix = "\n";
+ resultTables.add(tuple);
+ }
+ expectedOutput += "\n(" + numRows + " rows, " + responseTime + " sec, " + numBytes + " B selected)\n";
+
+ ResultSet resultSet = new TajoMetaDataResultSet(
+ Arrays.asList("col1", "col2", "col3"),
+ Arrays.asList(TajoDataTypes.Type.TEXT, TajoDataTypes.Type.INT4, TajoDataTypes.Type.FLOAT8),
+ resultTables);
+
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ outputFormatter.printResult(writer, null, tableDesc, responseTime, resultSet);
+
+ assertEquals(expectedOutput, stringWriter.toString());
+ }
+}