You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/02/06 09:35:48 UTC
[11/11] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/55e46301
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/55e46301
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/55e46301
Branch: refs/heads/index_support
Commit: 55e46301ebe92d2bf65aaab26688f957f1cf03c7
Parents: 1c1dfa0 161ee9e
Author: Jihoon Son <ji...@apache.org>
Authored: Fri Feb 6 17:35:22 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri Feb 6 17:35:22 2015 +0900
----------------------------------------------------------------------
CHANGES | 16 +
pom.xml | 1 +
.../org/apache/tajo/algebra/AlterTable.java | 34 +-
.../apache/tajo/algebra/AlterTableOpType.java | 2 +-
tajo-cli/pom.xml | 337 +++++++++
.../org/apache/tajo/cli/tools/TajoAdmin.java | 459 ++++++++++++
.../org/apache/tajo/cli/tools/TajoDump.java | 202 ++++++
.../org/apache/tajo/cli/tools/TajoGetConf.java | 161 +++++
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 210 ++++++
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ++++++
.../cli/tsql/InvalidStatementException.java | 25 +
.../org/apache/tajo/cli/tsql/ParsedResult.java | 53 ++
.../org/apache/tajo/cli/tsql/SimpleParser.java | 387 ++++++++++
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 686 ++++++++++++++++++
.../tajo/cli/tsql/TajoCliOutputFormatter.java | 97 +++
.../apache/tajo/cli/tsql/TajoFileHistory.java | 41 ++
.../tsql/commands/ConnectDatabaseCommand.java | 72 ++
.../cli/tsql/commands/CopyrightCommand.java | 65 ++
.../cli/tsql/commands/DescFunctionCommand.java | 136 ++++
.../cli/tsql/commands/DescTableCommand.java | 155 ++++
.../tsql/commands/ExecExternalShellCommand.java | 124 ++++
.../tajo/cli/tsql/commands/ExitCommand.java | 52 ++
.../tajo/cli/tsql/commands/HdfsCommand.java | 58 ++
.../tajo/cli/tsql/commands/HelpCommand.java | 133 ++++
.../cli/tsql/commands/ListDatabaseCommand.java | 50 ++
.../tajo/cli/tsql/commands/SetCommand.java | 127 ++++
.../cli/tsql/commands/TajoAdminCommand.java | 58 ++
.../cli/tsql/commands/TajoGetConfCommand.java | 58 ++
.../cli/tsql/commands/TajoHAAdminCommand.java | 58 ++
.../cli/tsql/commands/TajoShellCommand.java | 129 ++++
.../tajo/cli/tsql/commands/UnsetCommand.java | 53 ++
.../tajo/cli/tsql/commands/VersionCommand.java | 49 ++
.../org/apache/tajo/cli/tools/TajoAdmin.java | 457 ------------
.../org/apache/tajo/cli/tools/TajoDump.java | 202 ------
.../org/apache/tajo/cli/tools/TajoGetConf.java | 161 -----
.../org/apache/tajo/cli/tools/TajoHAAdmin.java | 211 ------
.../cli/tsql/DefaultTajoCliOutputFormatter.java | 211 ------
.../cli/tsql/InvalidStatementException.java | 25 -
.../org/apache/tajo/cli/tsql/ParsedResult.java | 53 --
.../org/apache/tajo/cli/tsql/SimpleParser.java | 388 ----------
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 701 -------------------
.../tajo/cli/tsql/TajoCliOutputFormatter.java | 98 ---
.../apache/tajo/cli/tsql/TajoFileHistory.java | 41 --
.../tsql/commands/ConnectDatabaseCommand.java | 72 --
.../cli/tsql/commands/CopyrightCommand.java | 65 --
.../cli/tsql/commands/DescFunctionCommand.java | 136 ----
.../cli/tsql/commands/DescTableCommand.java | 155 ----
.../tsql/commands/ExecExternalShellCommand.java | 124 ----
.../tajo/cli/tsql/commands/ExitCommand.java | 52 --
.../tajo/cli/tsql/commands/HdfsCommand.java | 58 --
.../tajo/cli/tsql/commands/HelpCommand.java | 133 ----
.../cli/tsql/commands/ListDatabaseCommand.java | 50 --
.../tajo/cli/tsql/commands/SetCommand.java | 127 ----
.../cli/tsql/commands/TajoAdminCommand.java | 58 --
.../cli/tsql/commands/TajoGetConfCommand.java | 58 --
.../cli/tsql/commands/TajoHAAdminCommand.java | 58 --
.../cli/tsql/commands/TajoShellCommand.java | 129 ----
.../tajo/cli/tsql/commands/UnsetCommand.java | 53 --
.../tajo/cli/tsql/commands/VersionCommand.java | 49 --
.../apache/tajo/client/TajoHAClientUtil.java | 87 ---
.../java/org/apache/tajo/conf/TajoConf.java | 5 +-
tajo-core/pom.xml | 4 +
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 2 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 12 +
.../tajo/engine/function/builtin/Lead.java | 92 +++
.../tajo/engine/function/builtin/LeadDate.java | 49 ++
.../engine/function/builtin/LeadDouble.java | 49 ++
.../tajo/engine/function/builtin/LeadFloat.java | 49 ++
.../tajo/engine/function/builtin/LeadInt.java | 49 ++
.../tajo/engine/function/builtin/LeadLong.java | 49 ++
.../engine/function/builtin/LeadString.java | 49 ++
.../tajo/engine/function/builtin/LeadTime.java | 49 ++
.../engine/function/builtin/LeadTimestamp.java | 49 ++
.../apache/tajo/engine/function/window/Lag.java | 82 +++
.../tajo/engine/function/window/LagDate.java | 42 ++
.../tajo/engine/function/window/LagDouble.java | 42 ++
.../tajo/engine/function/window/LagFloat.java | 42 ++
.../tajo/engine/function/window/LagInt.java | 42 ++
.../tajo/engine/function/window/LagLong.java | 42 ++
.../tajo/engine/function/window/LagString.java | 42 ++
.../tajo/engine/function/window/LagTime.java | 42 ++
.../engine/function/window/LagTimestamp.java | 42 ++
.../apache/tajo/engine/parser/SQLAnalyzer.java | 82 ++-
.../engine/planner/physical/WindowAggExec.java | 2 +-
.../apache/tajo/querymaster/QueryMaster.java | 15 +-
.../tajo/webapp/QueryExecutorServlet.java | 2 -
.../tajo/worker/ExecutionBlockContext.java | 28 +-
.../main/java/org/apache/tajo/worker/Task.java | 47 +-
.../apache/tajo/worker/TaskAttemptContext.java | 2 +-
.../apache/tajo/worker/TaskRunnerManager.java | 10 +-
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 23 +
.../engine/function/TestBuiltinFunctions.java | 38 +
.../tajo/engine/parser/TestSQLAnalyzer.java | 236 +++++--
.../tajo/engine/query/TestWindowQuery.java | 98 +++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 42 ++
.../apache/tajo/querymaster/TestKillQuery.java | 34 +
.../queries/TestWindowQuery/testLag1.sql | 18 +
.../TestWindowQuery/testLagWithDefault.sql | 18 +
.../TestWindowQuery/testLagWithNoArgs.sql | 18 +
.../queries/TestWindowQuery/testLead1.sql | 18 +
.../TestWindowQuery/testLeadWithDefault.sql | 18 +
.../TestWindowQuery/testLeadWithNoArgs.sql | 18 +
.../default/alter_table_add_partition_1.sql | 1 +
.../default/alter_table_add_partition_2.sql | 1 +
.../default/alter_table_add_partition_3.sql | 2 +
.../default/alter_table_add_partition_4.sql | 1 +
.../default/alter_table_drop_partition_1.sql | 1 +
.../default/alter_table_drop_partition_2.sql | 1 +
.../default/alter_table_drop_partition_3.sql | 1 +
.../testAlterTableAddPartition.result | 2 +
.../testAlterTableDropPartition.result | 2 +
.../results/TestWindowQuery/testLag1.result | 7 +
.../TestWindowQuery/testLagWithDefault.result | 7 +
.../TestWindowQuery/testLagWithNoArgs.result | 7 +
.../results/TestWindowQuery/testLead1.result | 7 +
.../TestWindowQuery/testLeadWithDefault.result | 7 +
.../TestWindowQuery/testLeadWithNoArgs.result | 7 +
tajo-dist/pom.xml | 1 +
.../configuration/catalog_configuration.rst | 12 +-
.../org/apache/tajo/plan/ExprAnnotator.java | 6 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 4 +-
.../org/apache/tajo/plan/TypeDeterminant.java | 4 +
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 22 +
.../plan/verifier/PreLogicalPlanVerifier.java | 12 +
tajo-project/pom.xml | 5 +
125 files changed, 5985 insertions(+), 4138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
----------------------------------------------------------------------
diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
index 0000000,f48a5b4..2395043
mode 000000,100644..100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
@@@ -1,0 -1,686 +1,686 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.tajo.cli.tsql;
+
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.base.Preconditions;
+ import com.google.protobuf.ServiceException;
+ import jline.UnsupportedTerminal;
+ import jline.console.ConsoleReader;
+ import org.apache.commons.cli.*;
+ import org.apache.tajo.*;
+ import org.apache.tajo.TajoProtos.QueryState;
+ import org.apache.tajo.catalog.TableDesc;
+ import org.apache.tajo.cli.tsql.ParsedResult.StatementType;
+ import org.apache.tajo.cli.tsql.SimpleParser.ParsingState;
+ import org.apache.tajo.cli.tsql.commands.*;
+ import org.apache.tajo.client.*;
+ import org.apache.tajo.conf.TajoConf;
+ import org.apache.tajo.conf.TajoConf.ConfVars;
+ import org.apache.tajo.ipc.ClientProtos;
+ import org.apache.tajo.util.FileUtil;
+
+ import java.io.*;
+ import java.lang.reflect.Constructor;
+ import java.sql.ResultSet;
+ import java.sql.SQLException;
+ import java.util.Collection;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.TreeMap;
+
+ public class TajoCli {
+ public static final String ERROR_PREFIX = "ERROR: ";
+ public static final String KILL_PREFIX = "KILL: ";
+
+ private final TajoConf conf;
+ private TajoClient client;
+ private final TajoCliContext context;
+
+ // Jline and Console related things
+ private final ConsoleReader reader;
+ private final InputStream sin;
+ private final PrintWriter sout;
+ private TajoFileHistory history;
+
+ // Current States
+ private String currentDatabase;
+
+ private TajoCliOutputFormatter displayFormatter;
+
+ private boolean wasError = false;
+
+ private static final Class [] registeredCommands = {
+ DescTableCommand.class,
+ DescFunctionCommand.class,
+ HelpCommand.class,
+ ExitCommand.class,
+ CopyrightCommand.class,
+ VersionCommand.class,
+ ConnectDatabaseCommand.class,
+ ListDatabaseCommand.class,
+ SetCommand.class,
+ UnsetCommand.class,
+ ExecExternalShellCommand.class,
+ HdfsCommand.class,
+ TajoAdminCommand.class,
+ TajoGetConfCommand.class,
+ TajoHAAdminCommand.class
+ };
+ private final Map<String, TajoShellCommand> commands = new TreeMap<String, TajoShellCommand>();
+
+ protected static final Options options;
+ private static final String HOME_DIR = System.getProperty("user.home");
+ private static final String HISTORY_FILE = ".tajo_history";
+
+ static {
+ options = new Options();
+ options.addOption("c", "command", true, "execute only single command, then exit");
+ options.addOption("f", "file", true, "execute commands from file, then exit");
+ options.addOption("h", "host", true, "Tajo server host");
+ options.addOption("p", "port", true, "Tajo server port");
+ options.addOption("B", "background", false, "execute as background process");
+ options.addOption("conf", "conf", true, "configuration value");
+ options.addOption("param", "param", true, "parameter value in SQL file");
+ options.addOption("help", "help", false, "help");
+ }
+
+ public class TajoCliContext extends OverridableConf {
+ public TajoCliContext(TajoConf conf) {
+ super(conf, ConfigKey.ConfigType.SESSION);
+ }
+
+ public TajoClient getTajoClient() {
+ return client;
+ }
+
+ public void setCurrentDatabase(String databasae) {
+ currentDatabase = databasae;
+ }
+
+ public String getCurrentDatabase() {
+ return currentDatabase;
+ }
+
+ public PrintWriter getOutput() {
+ return sout;
+ }
+
+ public TajoConf getConf() {
+ return conf;
+ }
+
+ @VisibleForTesting
+ public String getCliSideVar(String key) {
+ if (SessionVars.exists(key)) {
+ ConfigKey configKey = SessionVars.get(key);
+ return get(configKey);
+ } else {
+ return get(key);
+ }
+ }
+
+ public void setCliSideVar(String key, String value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+
+ boolean shouldReloadFormatter = false;
+
+ if (SessionVars.exists(key)) {
+ SessionVars configKey = SessionVars.get(key);
+ put(configKey, value);
+ shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR;
+ } else {
+ set(key, value);
+
+ // It is hard to recognize it is a client side variable. So, we always reload formatter.
+ shouldReloadFormatter = true;
+ }
+
+ if (shouldReloadFormatter) {
+ try {
+ initFormatter();
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + e.getMessage());
+ }
+ }
+ }
+
+ public Map<String, TajoShellCommand> getCommands() {
+ return commands;
+ }
+ }
+
+ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception {
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmd = parser.parse(options, args);
+
+ this.conf = new TajoConf(c);
+ context = new TajoCliContext(conf);
+ this.sin = in;
+ if (cmd.hasOption("B")) {
+ this.reader = new ConsoleReader(sin, out, new UnsupportedTerminal());
+ } else {
+ this.reader = new ConsoleReader(sin, out);
+ }
+
+ this.reader.setExpandEvents(false);
+ this.sout = new PrintWriter(reader.getOutput());
+ initFormatter();
+
+ if (cmd.hasOption("help")) {
+ printUsage();
+ System.exit(0);
+ }
+
+ String hostName = null;
+ Integer port = null;
+ if (cmd.hasOption("h")) {
+ hostName = cmd.getOptionValue("h");
+ }
+ if (cmd.hasOption("p")) {
+ port = Integer.parseInt(cmd.getOptionValue("p"));
+ }
+
+ String baseDatabase = null;
+ if (cmd.getArgList().size() > 0) {
+ baseDatabase = (String) cmd.getArgList().get(0);
+ }
+
+ if (cmd.getOptionValues("conf") != null) {
+ processConfVarCommand(cmd.getOptionValues("conf"));
+ }
+
+ // if there is no "-h" option,
+ if(hostName == null) {
+ if (conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
+ // it checks if the client service address is given in configuration and distributed mode.
+ // if so, it sets entryAddr.
+ hostName = conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[0];
+ }
+ }
+ if (port == null) {
+ if (conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS) != null) {
+ // it checks if the client service address is given in configuration and distributed mode.
+ // if so, it sets entryAddr.
+ port = Integer.parseInt(conf.getVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]);
+ }
+ }
+
+ if ((hostName == null) ^ (port == null)) {
+ System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address");
+ throw new RuntimeException("cannot find valid Tajo server address");
+ } else if (hostName != null && port != null) {
+ conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port);
+ client = new TajoClientImpl(conf, baseDatabase);
+ } else if (hostName == null && port == null) {
+ client = new TajoClientImpl(conf, baseDatabase);
+ }
+
+ try {
+ context.setCurrentDatabase(client.getCurrentDatabase());
+ initHistory();
+ initCommands();
+
+ if (cmd.getOptionValues("conf") != null) {
+ processSessionVarCommand(cmd.getOptionValues("conf"));
+ }
+
+ if (cmd.hasOption("c")) {
+ displayFormatter.setScriptMode();
+ int exitCode = executeScript(cmd.getOptionValue("c"));
+ sout.flush();
+ System.exit(exitCode);
+ }
+ if (cmd.hasOption("f")) {
+ displayFormatter.setScriptMode();
+ cmd.getOptionValues("");
+ File sqlFile = new File(cmd.getOptionValue("f"));
+ if (sqlFile.exists()) {
+ String script = FileUtil.readTextFile(new File(cmd.getOptionValue("f")));
+ script = replaceParam(script, cmd.getOptionValues("param"));
+ int exitCode = executeScript(script);
+ sout.flush();
+ System.exit(exitCode);
+ } else {
+ System.err.println(ERROR_PREFIX + "No such a file \"" + cmd.getOptionValue("f") + "\"");
+ System.exit(-1);
+ }
+ }
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + "Exception was thrown. Caused by " + e.getMessage());
+
+ if (client != null) {
+ client.close();
+ }
+
+ throw e;
+ }
+
+ addShutdownHook();
+ }
+
+ private void processConfVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (!SessionVars.exists(tokens[0])) {
+ conf.set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
+ private void processSessionVarCommand(String[] confCommands) throws ServiceException {
+ for (String eachParam: confCommands) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+
+ if (SessionVars.exists(tokens[0])) {
+ ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]);
+ }
+ }
+ }
+
+ private void initFormatter() throws Exception {
+ Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS);
+ if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) {
+ displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance();
+ }
+ displayFormatter.init(context);
+ }
+
+ public TajoCliContext getContext() {
+ return context;
+ }
+
+ protected static String replaceParam(String script, String[] params) {
+ if (params == null || params.length == 0) {
+ return script;
+ }
+
+ for (String eachParam: params) {
+ String[] tokens = eachParam.split("=");
+ if (tokens.length != 2) {
+ continue;
+ }
+ script = script.replace("${" + tokens[0] + "}", tokens[1]);
+ }
+
+ return script;
+ }
+
+ private void initHistory() {
+ try {
+ String historyPath = HOME_DIR + File.separator + HISTORY_FILE;
+ if ((new File(HOME_DIR)).exists()) {
+ history = new TajoFileHistory(new File(historyPath));
+ history.setAutoTrim(false);
+ history.setIgnoreDuplicates(false);
+ reader.setHistory(history);
+ } else {
+ System.err.println(ERROR_PREFIX + "home directory : '" + HOME_DIR +"' does not exist.");
+ }
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + e.getMessage());
+ }
+ }
+
+ private void initCommands() {
+ for (Class clazz : registeredCommands) {
+ TajoShellCommand cmd = null;
+ try {
+ Constructor cons = clazz.getConstructor(new Class[] {TajoCliContext.class});
+ cmd = (TajoShellCommand) cons.newInstance(context);
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ throw new RuntimeException(e.getMessage());
+ }
+ commands.put(cmd.getCommand(), cmd);
+ for (String alias : cmd.getAliases()) {
+ commands.put(alias, cmd);
+ }
+ }
+ }
+
+ private void addShutdownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ history.flush();
+ } catch (IOException e) {
+ }
+ client.close();
+ }
+ }));
+ }
+
+ private String updatePrompt(ParsingState state) throws ServiceException {
+ if (state == ParsingState.WITHIN_QUOTE) {
+ return "'";
+ } else if (state == ParsingState.TOK_START) {
+ return context.getCurrentDatabase();
+ } else {
+ return "";
+ }
+ }
+
+ public int runShell() throws Exception {
+ String line;
+ String currentPrompt = context.getCurrentDatabase();
+ int exitCode;
+ ParsingState latestState = SimpleParser.START_STATE;
+
+ sout.write("Try \\? for help.\n");
+
+ SimpleParser parser = new SimpleParser();
+
+ try {
+ while((line = reader.readLine(currentPrompt + "> ")) != null) {
+ if (line.equals("")) {
+ continue;
+ }
+ wasError = false;
+ if (line.startsWith("{")) {
+ executeJsonQuery(line);
+ } else {
+ List<ParsedResult> parsedResults = parser.parseLines(line);
+
+ if (latestState != ParsingState.TOK_START && parsedResults.size() > 0) {
+ // Add multi-line statements to history in addition to individual lines.
+ ParsedResult parsed = parsedResults.get(0);
+ history.add(parsed.getHistoryStatement() + (parsed.getType() == StatementType.STATEMENT ? ";" : ""));
+ }
+
+ exitCode = executeParsedResults(parsedResults);
+ latestState = parser.getState();
+ currentPrompt = updatePrompt(latestState);
+
+ // if at least one failed
+ if (exitCode != 0) {
+ return exitCode;
+ }
+ }
+ }
+ } catch (Exception e) {
+ System.err.println(ERROR_PREFIX + "Exception was thrown. Casued by " + e.getMessage());
+
+ if (client != null) {
+ client.close();
+ }
+
+ throw e;
+ }
+ return 0;
+ }
+
+ private int executeParsedResults(Collection<ParsedResult> parsedResults) throws Exception {
+ int exitCode;
+ for (ParsedResult parsedResult : parsedResults) {
+ if (parsedResult.getType() == StatementType.META) {
+ exitCode = executeMetaCommand(parsedResult.getStatement());
+ } else {
+ exitCode = executeQuery(parsedResult.getStatement());
+ }
+
+ if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) {
+ return exitCode;
+ }
+ }
+
+ return 0;
+ }
+
+ public int executeMetaCommand(String line) throws Exception {
+ String [] metaCommands = line.split(";");
+ for (String metaCommand : metaCommands) {
+ String arguments [] = metaCommand.split(" ");
+
+ TajoShellCommand invoked = commands.get(arguments[0]);
+ if (invoked == null) {
+ printInvalidCommand(arguments[0]);
+ wasError = true;
+ return -1;
+ }
+
+ try {
+ invoked.invoke(arguments);
+ } catch (IllegalArgumentException ige) {
+ displayFormatter.printErrorMessage(sout, ige);
+ wasError = true;
+ return -1;
+ } catch (Exception e) {
+ displayFormatter.printErrorMessage(sout, e);
+ wasError = true;
+ return -1;
+ } finally {
+ context.getOutput().flush();
+ }
+
+ if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) {
+ break;
+ }
+ }
+
+ return 0;
+ }
+
+ private void executeJsonQuery(String json) throws ServiceException, IOException {
+
+ long startTime = System.currentTimeMillis();
+ ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json);
+ if (response == null) {
+ displayFormatter.printErrorMessage(sout, "response is null");
+ wasError = true;
- } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
++ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
+ if (response.getIsForwarded()) {
+ QueryId queryId = new QueryId(response.getQueryId());
+ waitForQueryCompleted(queryId);
+ } else {
+ if (!response.hasTableDesc() && !response.hasResultSet()) {
+ displayFormatter.printMessage(sout, "OK");
+ wasError = true;
+ } else {
+ localQueryCompleted(response, startTime);
+ }
+ }
+ } else {
- if (response.hasErrorMessage()) {
- displayFormatter.printErrorMessage(sout, response.getErrorMessage());
++ if (response.getResult().hasErrorMessage()) {
++ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage());
+ wasError = true;
+ }
+ }
+ }
+
+ private int executeQuery(String statement) throws ServiceException, IOException {
+
+ long startTime = System.currentTimeMillis();
+ ClientProtos.SubmitQueryResponse response = null;
+ try{
+ response = client.executeQuery(statement);
+ } catch (ServiceException e){
+ displayFormatter.printErrorMessage(sout, e.getMessage());
+ wasError = true;
+ } catch(Throwable te){
+ displayFormatter.printErrorMessage(sout, te);
+ wasError = true;
+ }
+
+ if (response == null) {
+ displayFormatter.printErrorMessage(sout, "response is null");
+ wasError = true;
- } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
++ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
+ if (response.getIsForwarded()) {
+ QueryId queryId = new QueryId(response.getQueryId());
+ waitForQueryCompleted(queryId);
+ } else {
+ if (!response.hasTableDesc() && !response.hasResultSet()) {
+ displayFormatter.printMessage(sout, "OK");
+ } else {
+ localQueryCompleted(response, startTime);
+ }
+ }
+ } else {
- if (response.hasErrorMessage()) {
- displayFormatter.printErrorMessage(sout, response.getErrorMessage());
++ if (response.getResult().hasErrorMessage()) {
++ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage());
+ wasError = true;
+ }
+ }
+
+ return wasError ? -1 : 0;
+ }
+
+ private void localQueryCompleted(ClientProtos.SubmitQueryResponse response, long startTime) {
+ ResultSet res = null;
+ try {
+ QueryId queryId = new QueryId(response.getQueryId());
+ float responseTime = ((float)(System.currentTimeMillis() - startTime) / 1000.0f);
+ TableDesc desc = new TableDesc(response.getTableDesc());
+
+ // non-forwarded INSERT INTO query does not have any query id.
+ // In this case, it just returns succeeded query information without printing the query results.
+ if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
+ } else {
+ res = TajoClientUtil.createResultSet(conf, client, response);
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
+ }
+ } catch (Throwable t) {
+ displayFormatter.printErrorMessage(sout, t);
+ wasError = true;
+ } finally {
+ if (res != null) {
+ try {
+ res.close();
+ } catch (SQLException e) {
+ }
+ }
+ }
+ }
+
+ private void waitForQueryCompleted(QueryId queryId) {
+ // if query is empty string
+ if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
+ return;
+ }
+
+ // query execute
+ ResultSet res = null;
+ QueryStatus status = null;
+ try {
+
+ int initRetries = 0;
+ int progressRetries = 0;
+ while (true) {
+ // TODO - configurable
+ status = client.getQueryStatus(queryId);
+ if(TajoClientUtil.isQueryWaitingForSchedule(status.getState())) {
+ Thread.sleep(Math.min(20 * initRetries, 1000));
+ initRetries++;
+ continue;
+ }
+
+ if (TajoClientUtil.isQueryRunning(status.getState()) || status.getState() == QueryState.QUERY_SUCCEEDED) {
+ displayFormatter.printProgress(sout, status);
+ }
+
+ if (TajoClientUtil.isQueryComplete(status.getState()) && status.getState() != QueryState.QUERY_KILL_WAIT) {
+ break;
+ } else {
+ Thread.sleep(Math.min(200 * progressRetries, 1000));
+ progressRetries += 2;
+ }
+ }
+
+ if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) {
+ displayFormatter.printErrorMessage(sout, status);
+ wasError = true;
+ } else if (status.getState() == QueryState.QUERY_KILLED) {
+ displayFormatter.printKilledMessage(sout, queryId);
+ wasError = true;
+ } else {
+ if (status.getState() == QueryState.QUERY_SUCCEEDED) {
+ float responseTime = ((float)(status.getFinishTime() - status.getSubmitTime()) / 1000.0f);
+ ClientProtos.GetQueryResultResponse response = client.getResultResponse(queryId);
+ if (status.hasResult()) {
+ res = TajoClientUtil.createResultSet(conf, client, queryId, response);
+ TableDesc desc = new TableDesc(response.getTableDesc());
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
+ } else {
+ TableDesc desc = new TableDesc(response.getTableDesc());
+ displayFormatter.printResult(sout, sin, desc, responseTime, res);
+ }
+ }
+ }
+ } catch (Throwable t) {
+ displayFormatter.printErrorMessage(sout, t);
+ wasError = true;
+ } finally {
+ if (res != null) {
+ try {
+ res.close();
+ } catch (SQLException e) {
+ }
+ } else {
+ if (status != null && status.getQueryId() != null) {
+ client.closeQuery(status.getQueryId());
+ }
+ }
+ }
+ }
+
+ public int executeScript(String script) throws Exception {
+ wasError = false;
+ List<ParsedResult> results = SimpleParser.parseScript(script);
+ return executeParsedResults(results);
+ }
+
+ private void printUsage() {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("tsql [options] [database]", options);
+ }
+
+ private void printInvalidCommand(String command) {
+ sout.println("Invalid command " + command + ". Try \\? for help.");
+ }
+
+ @VisibleForTesting
+ public void close() {
+ //for testcase
+ if (client != null) {
+ client.close();
+ }
+
+ if (reader != null) {
+ reader.shutdown();
+ }
+ }
+
+ public static void main(String [] args) throws Exception {
+ TajoConf conf = new TajoConf();
+ TajoCli shell = new TajoCli(conf, args, System.in, System.out);
+ System.out.println();
+ System.exit(shell.runShell());
+ }
+ }
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --cc tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index 0000000,b7d9334..cae7018
mode 000000,100644..100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@@ -1,0 -1,137 +1,155 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.tajo.cli.tsql.commands;
+
+ import org.apache.commons.lang.CharUtils;
+ import org.apache.commons.lang.StringEscapeUtils;
+ import org.apache.tajo.TajoConstants;
+ import org.apache.tajo.catalog.CatalogUtil;
+ import org.apache.tajo.catalog.Column;
+ import org.apache.tajo.catalog.TableDesc;
+ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
++import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
++import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
+ import org.apache.tajo.cli.tsql.TajoCli;
+ import org.apache.tajo.util.FileUtil;
+ import org.apache.tajo.util.TUtil;
+
+ import java.util.List;
+ import java.util.Map;
+
+ public class DescTableCommand extends TajoShellCommand {
+ public DescTableCommand(TajoCli.TajoCliContext context) {
+ super(context);
+ }
+
+ @Override
+ public String getCommand() {
+ return "\\d";
+ }
+
+ @Override
+ public void invoke(String[] cmd) throws Exception {
+ if (cmd.length == 2) {
+ String tableName = cmd[1];
+ tableName = tableName.replace("\"", "");
+ TableDesc desc = client.getTableDesc(tableName);
+ if (desc == null) {
+ context.getOutput().println("Did not find any relation named \"" + tableName + "\"");
+ } else {
+ context.getOutput().println(toFormattedString(desc));
++ // If there exists any indexes for the table, print index information
++ if (client.hasIndexes(tableName)) {
++ StringBuilder sb = new StringBuilder();
++ sb.append("Indexes:\n");
++ for (IndexDescProto index : client.getIndexes(tableName)) {
++ sb.append("\"").append(index.getIndexName()).append("\" ");
++ sb.append(index.getIndexMethod()).append(" (");
++ for (SortSpecProto key : index.getKeySortSpecsList()) {
++ sb.append(CatalogUtil.extractSimpleName(key.getColumn().getName()));
++ sb.append(key.getAscending() ? " ASC" : " DESC");
++ sb.append(key.getNullFirst() ? " NULLS FIRST, " : " NULLS LAST, ");
++ }
++ sb.delete(sb.length()-2, sb.length()-1).append(")\n");
++ }
++ context.getOutput().println(sb.toString());
++ }
+ }
+ } else if (cmd.length == 1) {
+ List<String> tableList = client.getTableList(null);
+ if (tableList.size() == 0) {
+ context.getOutput().println("No Relation Found");
+ }
+ for (String table : tableList) {
+ context.getOutput().println(table);
+ }
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return "[table_name]";
+ }
+
+ @Override
+ public String getDescription() {
+ return "show table description";
+ }
+
+ protected String toFormattedString(TableDesc desc) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\ntable name: ").append(desc.getName()).append("\n");
+ sb.append("table path: ").append(desc.getPath()).append("\n");
+ sb.append("store type: ").append(CatalogUtil.getStoreTypeString(desc.getMeta().getStoreType())).append("\n");
+ if (desc.getStats() != null) {
+
+ long row = desc.getStats().getNumRows();
+ String rowText = row == TajoConstants.UNKNOWN_ROW_NUMBER ? "unknown" : row + "";
+ sb.append("number of rows: ").append(rowText).append("\n");
+ sb.append("volume: ").append(
+ FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(),
+ true)).append("\n");
+ }
+ sb.append("Options: \n");
+ for(Map.Entry<String, String> entry : desc.getMeta().toMap().entrySet()){
+
+ /*
+ * Checks whether the character is ASCII 7 bit printable.
+ * For example, a printable unicode '\u007c' become the character ‘|’.
+ *
+ * Control-chars : ctrl-a(\u0001), tab(\u0009) ..
+ * Printable-chars : '|'(\u007c), ','(\u002c) ..
+ * */
+
+ String value = entry.getValue();
+ String unescaped = StringEscapeUtils.unescapeJava(value);
+ if (unescaped.length() == 1 && CharUtils.isAsciiPrintable(unescaped.charAt(0))) {
+ value = unescaped;
+ }
+ sb.append("\t").append("'").append(entry.getKey()).append("'").append("=")
+ .append("'").append(value).append("'").append("\n");
+ }
+ sb.append("\n");
+ sb.append("schema: \n");
+
+ for(int i = 0; i < desc.getSchema().size(); i++) {
+ Column col = desc.getSchema().getColumn(i);
+ sb.append(col.getSimpleName()).append("\t").append(col.getDataType().getType());
+ if (col.getDataType().hasLength()) {
+ sb.append("(").append(col.getDataType().getLength()).append(")");
+ }
+ sb.append("\n");
+ }
+
+ sb.append("\n");
+ if (desc.getPartitionMethod() != null) {
+ PartitionMethodDesc partition = desc.getPartitionMethod();
+ sb.append("Partitions: \n");
+
+ sb.append("type:").append(partition.getPartitionType().name()).append("\n");
+
+ sb.append("columns:").append(":");
+ sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray()));
+ }
+
+ return sb.toString();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 14b0d1a,8f84a9d..859b3ae
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@@ -120,12 -129,27 +129,15 @@@ public class Task
request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir);
this.context.setDataChannel(request.getDataChannel());
this.context.setEnforcer(request.getEnforcer());
+ this.context.setState(TaskAttemptState.TA_PENDING);
this.inputStats = new TableStats();
+ }
+ public void initPlan() throws IOException {
plan = LogicalNodeDeserializer.deserialize(queryContext, request.getPlan());
- LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN);
- if (scanNode != null) {
- for (LogicalNode node : scanNode) {
- ScanNode scan = (ScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
-
- LogicalNode [] partitionScanNode = PlannerUtil.findAllNodes(plan, NodeType.PARTITIONS_SCAN);
- if (partitionScanNode != null) {
- for (LogicalNode node : partitionScanNode) {
- PartitionedTableScanNode scan = (PartitionedTableScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
+ updateDescsForScanNodes(NodeType.SCAN);
+ updateDescsForScanNodes(NodeType.PARTITIONS_SCAN);
+ updateDescsForScanNodes(NodeType.INDEX_SCAN);
interQuery = request.getProto().getInterQuery();
if (interQuery) {
@@@ -167,18 -189,9 +177,20 @@@
LOG.info("==================================");
}
+ private void updateDescsForScanNodes(NodeType nodeType) {
+ assert nodeType == NodeType.SCAN || nodeType == NodeType.PARTITIONS_SCAN || nodeType == NodeType.INDEX_SCAN;
+ LogicalNode[] scanNodes = PlannerUtil.findAllNodes(plan, nodeType);
+ if (scanNodes != null) {
+ for (LogicalNode node : scanNodes) {
+ ScanNode scanNode = (ScanNode) node;
+ descs.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+ }
+ }
+ }
+
public void init() throws IOException {
+ initPlan();
+
if (context.getState() == TaskAttemptState.TA_PENDING) {
// initialize a task temporal dir
FileSystem localFS = executionBlockContext.getLocalFS();
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/55e46301/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------