You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/04/08 14:28:53 UTC
[2/2] git commit: TAJO-356: Improve TajoClient to directly get query
results in the first request. (hyunsik)
TAJO-356: Improve TajoClient to directly get query results in the first request. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/631f3d04
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/631f3d04
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/631f3d04
Branch: refs/heads/master
Commit: 631f3d04a20529ad89641757ca2370de27d6c935
Parents: 8b5361a
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Apr 8 21:27:35 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Apr 8 21:27:35 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../java/org/apache/tajo/algebra/Explain.java | 38 ++++
.../java/org/apache/tajo/algebra/OpType.java | 1 +
.../org/apache/tajo/catalog/CatalogUtil.java | 7 +-
.../apache/tajo/catalog/TestCatalogUtil.java | 21 +++
.../java/org/apache/tajo/cli/SimpleParser.java | 2 +
.../main/java/org/apache/tajo/cli/TajoCli.java | 171 +++++++++--------
.../java/org/apache/tajo/client/TajoClient.java | 80 +++++---
.../apache/tajo/jdbc/TajoMemoryResultSet.java | 79 ++++++++
.../org/apache/tajo/jdbc/TajoResultSet.java | 39 +++-
tajo-client/src/main/proto/ClientProtos.proto | 28 ++-
.../main/proto/TajoMasterClientProtocol.proto | 3 +-
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 2 +-
.../org/apache/tajo/engine/parser/SQLParser.g4 | 12 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 7 +-
.../tajo/engine/planner/AlgebraVisitor.java | 1 +
.../tajo/engine/planner/BaseAlgebraVisitor.java | 11 ++
.../apache/tajo/engine/planner/LogicalPlan.java | 12 +-
.../tajo/engine/planner/LogicalPlanner.java | 5 +
.../apache/tajo/engine/planner/PlannerUtil.java | 59 ++++++
.../org/apache/tajo/engine/utils/TupleUtil.java | 3 +-
.../org/apache/tajo/master/GlobalEngine.java | 184 +++++++++++++------
.../tajo/master/TajoMasterClientService.java | 31 +---
.../tajo/master/querymaster/Repartitioner.java | 2 +-
.../tajo/webapp/QueryExecutorServlet.java | 2 +-
.../tajo/worker/RangeRetrieverHandler.java | 3 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 4 +-
.../org/apache/tajo/cli/TestSimpleParser.java | 15 ++
.../org/apache/tajo/client/TestTajoClient.java | 10 +-
.../planner/physical/TestPhysicalPlanner.java | 2 +-
.../tajo/engine/query/TestSelectQuery.java | 32 ++++
.../apache/tajo/engine/util/TestTupleUtil.java | 9 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 2 +-
.../TestSelectQuery/testExplainSelect.sql | 1 +
.../TestSelectQuery/testNonFromSelect1.sql | 1 +
.../queries/TestSelectQuery/testSimpleQuery.sql | 1 +
.../testSimpleQueryWithLimit.sql | 1 +
.../TestSelectQuery/testExplainSelect.result | 6 +
.../TestSelectQuery/testNonFromSelect1.result | 3 +
.../TestSelectQuery/testSimpleQuery.result | 7 +
.../testSimpleQueryWithLimit.result | 5 +
.../tajo/pullserver/PullServerAuxService.java | 3 +-
.../tajo/pullserver/TajoPullServerService.java | 3 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 20 +-
.../apache/tajo/storage/index/bst/BSTIndex.java | 7 +-
45 files changed, 697 insertions(+), 241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9412ae4..3d205bb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -143,6 +143,9 @@ Release 0.8.0 - unreleased
IMPROVEMENTS
+ TAJO-356: Improve TajoClient to directly get query results in the first request.
+ (hyunsik)
+
TAJO-728: Supports expression IN statement. (hyunsik)
TAJO-725: Broadcast JOIN should supports multiple tables. (hyoungjunkim via jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java
new file mode 100644
index 0000000..ee76ea9
--- /dev/null
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Explain.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.algebra;
+
+import com.google.common.base.Objects;
+
+public class Explain extends UnaryOperator {
+
+ public Explain(Expr operand) {
+ super(OpType.Explain);
+ setChild(operand);
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(getChild());
+ }
+
+ @Override
+ boolean equalsTo(Expr expr) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
index a4fb617..0cb0527 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
@@ -40,6 +40,7 @@ public enum OpType {
RelationList(RelationList.class),
Relation(Relation.class),
ScalarSubQuery(ScalarSubQuery.class),
+ Explain(Explain.class),
// Data definition language
CreateDatabase(CreateDatabase.class),
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index f9f92f0..5bcb290 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -150,15 +150,18 @@ public class CatalogUtil {
boolean openQuote = identifier.charAt(0) == '"';
boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"';
- if (openQuote ^ closeQuote || identifier.length() < 2) {
+ // if at least one quote mark exists, the identifier must be grater than equal to 2 characters,
+ if (openQuote ^ closeQuote && identifier.length() < 2) {
throw new IllegalArgumentException("Invalid Identifier: " + identifier);
}
+ // does not allow the empty identifier (''),
if (openQuote && closeQuote && identifier.length() == 2) {
throw new IllegalArgumentException("zero-length delimited identifier: " + identifier);
}
- return openQuote && closeQuote && identifier.length() > 2;
+ // Ensure the quote open and close
+ return openQuote && closeQuote;
}
public static boolean isFQColumnName(String tableName) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
index f2d9f89..75149e9 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java
@@ -30,4 +30,25 @@ public class TestCatalogUtil {
Type.INT8));
assertEquals("sum(int4,int8)", canonical);
}
+
+ String [] sources = {
+ "A",
+ "Column_Name",
+ "COLUMN_NAME",
+ "컬럼"
+ };
+
+ String [] normalized = {
+ "a",
+ "column_name",
+ "column_name",
+ "컬럼"
+ };
+
+ @Test
+ public final void testNormalizeIdentifier() {
+ for (int i = 0; i < sources.length; i++) {
+ assertEquals(normalized[i], CatalogUtil.normalizeIdentifier(sources[i]));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
index 2146df7..61f6d2a 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
@@ -178,6 +178,8 @@ public class SimpleParser {
appender.append(" ");
}
}
+ } else { // skip unknown character
+ idx++;
}
lineNum++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
index 2a49d0b..c277c4c 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -24,21 +24,19 @@ import org.apache.commons.cli.*;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.TajoProtos.QueryState;
-import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.QueryStatus;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.ipc.ClientProtos;
-import org.apache.tajo.jdbc.TajoResultSet;
import org.apache.tajo.util.FileUtil;
import java.io.*;
import java.lang.reflect.Constructor;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -302,23 +300,40 @@ public class TajoCli {
}
private void executeQuery(String statement) throws ServiceException {
- ClientProtos.GetQueryStatusResponse response = client.executeQuery(statement);
+ ClientProtos.SubmitQueryResponse response = client.executeQuery(statement);
if (response == null) {
sout.println("response is null");
- }
- else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
- QueryId queryId = null;
- try {
- queryId = new QueryId(response.getQueryId());
- if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- sout.println("OK");
- } else {
+ } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (response.getIsForwarded()) {
+ QueryId queryId = new QueryId(response.getQueryId());
+ try {
waitForQueryCompleted(queryId);
- }
- } finally {
- if(queryId != null) {
+ } finally {
client.closeQuery(queryId);
}
+ } else {
+ if (!response.hasTableDesc() && !response.hasResultSet()) {
+ sout.println("Ok");
+ } else {
+
+ ResultSet resultSet;
+ int numBytes;
+ long maxRowNum;
+ try {
+ resultSet = TajoClient.createResultSet(client, response);
+ if (response.hasTableDesc()) {
+ numBytes = 0;
+ } else {
+ numBytes = response.getResultSet().getBytesNum();
+ }
+ maxRowNum = response.getMaxRowNum();
+ printResult(resultSet, maxRowNum, numBytes);
+ } catch (IOException ioe) {
+ sout.println(ioe.getMessage());
+ } catch (SQLException sqe) {
+ sout.println(sqe.getMessage());
+ }
+ }
}
} else {
if (response.hasErrorMessage()) {
@@ -340,7 +355,7 @@ public class TajoCli {
int initRetries = 0;
int progressRetries = 0;
while (true) {
- // TODO - configurabl
+ // TODO - configurable
status = client.getQueryStatus(queryId);
if(status.getState() == QueryState.QUERY_MASTER_INIT || status.getState() == QueryState.QUERY_MASTER_LAUNCHED) {
Thread.sleep(Math.min(20 * initRetries, 1000));
@@ -379,68 +394,12 @@ public class TajoCli {
+ ", response time: " + (((float)(status.getFinishTime() - status.getSubmitTime()) / 1000.0)
+ " sec"));
if (status.hasResult()) {
- ResultSet res = null;
- TableDesc desc = null;
- if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- res = client.createNullResultSet(queryId);
- } else {
- ClientProtos.GetQueryResultResponse response = client.getResultResponse(queryId);
- desc = CatalogUtil.newTableDesc(response.getTableDesc());
- conf.setVar(ConfVars.USERNAME, response.getTajoUserName());
- res = new TajoResultSet(client, queryId, conf, desc);
- }
- try {
- if (res == null) {
- sout.println("OK");
- return;
- }
-
- ResultSetMetaData rsmd = res.getMetaData();
-
- TableStats stat = desc.getStats();
- String volume = FileUtil.humanReadableByteCount(stat.getNumBytes(), false);
- long resultRows = stat.getNumRows();
- sout.println("result: " + desc.getPath() + ", " + resultRows + " rows (" + volume + ")");
-
- int numOfColumns = rsmd.getColumnCount();
- for (int i = 1; i <= numOfColumns; i++) {
- if (i > 1) sout.print(", ");
- String columnName = rsmd.getColumnName(i);
- sout.print(columnName);
- }
- sout.println("\n-------------------------------");
-
- int numOfPrintedRows = 0;
- while (res.next()) {
- // TODO - to be improved to print more formatted text
- for (int i = 1; i <= numOfColumns; i++) {
- if (i > 1) sout.print(", ");
- String columnValue = res.getObject(i).toString();
- if(res.wasNull()){
- sout.print("null");
- } else {
- sout.print(columnValue);
- }
- }
- sout.println();
- sout.flush();
- numOfPrintedRows++;
- if (numOfPrintedRows >= PRINT_LIMIT) {
- sout.print("continue... ('q' is quit)");
- sout.flush();
- if (sin.read() == 'q') {
- sout.println();
- break;
- }
- numOfPrintedRows = 0;
- sout.println();
- }
- }
- } finally {
- if(res != null) {
- res.close();
- }
- }
+ ClientProtos.GetQueryResultResponse response = client.getResultResponse(queryId);
+ ResultSet res = TajoClient.createResultSet(client, queryId, response);
+ TableDesc desc = new TableDesc(response.getTableDesc());
+ long totalRowNum = desc.getStats().getNumRows();
+ long totalBytes = desc.getStats().getNumBytes();
+ printResult(res, totalRowNum, totalBytes);
} else {
sout.println("OK");
}
@@ -452,6 +411,62 @@ public class TajoCli {
}
}
+ private void printResult(ResultSet res, long rowNum, long numBytes) throws IOException, SQLException {
+ try {
+ if (res == null) {
+ sout.println("OK");
+ return;
+ }
+
+ ResultSetMetaData rsmd = res.getMetaData();
+
+ String volume = FileUtil.humanReadableByteCount(numBytes, false);
+ String rowNumStr = rowNum == Integer.MAX_VALUE ? "unknown" : rowNum + "";
+ sout.println("result: " + rowNumStr + " rows (" + volume + ")");
+
+ int numOfColumns = rsmd.getColumnCount();
+ for (int i = 1; i <= numOfColumns; i++) {
+ if (i > 1) sout.print(", ");
+ String columnName = rsmd.getColumnName(i);
+ sout.print(columnName);
+ }
+ sout.println("\n-------------------------------");
+
+ int numOfPrintedRows = 0;
+ while (res.next()) {
+ // TODO - to be improved to print more formatted text
+ for (int i = 1; i <= numOfColumns; i++) {
+ if (i > 1) sout.print(", ");
+ String columnValue = res.getObject(i).toString();
+ if(res.wasNull()){
+ sout.print("null");
+ } else {
+ sout.print(columnValue);
+ }
+ }
+ sout.println();
+ sout.flush();
+ numOfPrintedRows++;
+ if (numOfPrintedRows >= PRINT_LIMIT) {
+ sout.print("continue... ('q' is quit)");
+ sout.flush();
+ if (sin.read() == 'q') {
+ sout.println();
+ break;
+ }
+ numOfPrintedRows = 0;
+ sout.println();
+ }
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ } finally {
+ if(res != null) {
+ res.close();
+ }
+ }
+ }
+
public int executeScript(String script) throws Exception {
List<ParsedResult> results = SimpleParser.parseScript(script);
executeParsedResults(results);
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
index dc51b63..32d09df 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
@@ -40,6 +40,7 @@ import org.apache.tajo.ipc.QueryMasterClientProtocol.QueryMasterClientProtocolSe
import org.apache.tajo.ipc.TajoMasterClientProtocol;
import org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService;
import org.apache.tajo.jdbc.SQLStates;
+import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.jdbc.TajoResultSet;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
@@ -55,6 +56,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse.SerializedResultSet;
+
@ThreadSafe
public class TajoClient implements Closeable {
private final Log LOG = LogFactory.getLog(TajoClient.class);
@@ -108,6 +111,10 @@ public class TajoClient implements Closeable {
this(new TajoConf(), NetUtils.createSocketAddr(hostname, port), baseDatabase);
}
+ public TajoIdProtos.SessionIdProto getSessionId() {
+ return sessionId;
+ }
+
@Override
public void close() {
// remove session
@@ -274,28 +281,16 @@ public class TajoClient implements Closeable {
}.withRetries();
}
- public ExplainQueryResponse explainQuery(final String sql) throws ServiceException {
- return new ServerCallable<ExplainQueryResponse>(connPool, tajoMasterAddr,
- TajoMasterClientProtocol.class, false, true) {
- public ExplainQueryResponse call(NettyClientBase client) throws ServiceException {
- checkSessionAndGet(client);
-
- TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub();
- return tajoMasterService.explainQuery(null, convertSessionedString(sql));
- }
- }.withRetries();
- }
-
/**
* It submits a query statement and get a response immediately.
* The response only contains a query id, and submission status.
* In order to get the result, you should use {@link #getQueryResult(org.apache.tajo.QueryId)}
* or {@link #getQueryResultAndWait(org.apache.tajo.QueryId)}.
*/
- public GetQueryStatusResponse executeQuery(final String sql) throws ServiceException {
- return new ServerCallable<GetQueryStatusResponse>(connPool, tajoMasterAddr,
+ public SubmitQueryResponse executeQuery(final String sql) throws ServiceException {
+ return new ServerCallable<SubmitQueryResponse>(connPool, tajoMasterAddr,
TajoMasterClientProtocol.class, false, true) {
- public GetQueryStatusResponse call(NettyClientBase client) throws ServiceException {
+ public SubmitQueryResponse call(NettyClientBase client) throws ServiceException {
checkSessionAndGet(client);
final QueryRequest.Builder builder = QueryRequest.newBuilder();
@@ -317,9 +312,9 @@ public class TajoClient implements Closeable {
*/
public ResultSet executeQueryAndGetResult(final String sql)
throws ServiceException, IOException {
- GetQueryStatusResponse response = new ServerCallable<GetQueryStatusResponse>(connPool, tajoMasterAddr,
+ SubmitQueryResponse response = new ServerCallable<SubmitQueryResponse>(connPool, tajoMasterAddr,
TajoMasterClientProtocol.class, false, true) {
- public GetQueryStatusResponse call(NettyClientBase client) throws ServiceException {
+ public SubmitQueryResponse call(NettyClientBase client) throws ServiceException {
checkSessionAndGet(client);
final QueryRequest.Builder builder = QueryRequest.newBuilder();
@@ -331,10 +326,18 @@ public class TajoClient implements Closeable {
}.withRetries();
QueryId queryId = new QueryId(response.getQueryId());
- if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- return this.createNullResultSet(queryId);
+ if (response.getIsForwarded()) {
+ if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
+ return this.createNullResultSet(queryId);
+ } else {
+ return this.getQueryResultAndWait(queryId);
+ }
} else {
- return this.getQueryResultAndWait(queryId);
+ if (response.hasResultSet() || response.hasTableDesc()) {
+ return createResultSet(this, response);
+ } else {
+ return this.createNullResultSet(queryId);
+ }
}
}
@@ -411,7 +414,42 @@ public class TajoClient implements Closeable {
return new TajoResultSet(this, queryId, conf, tableDesc);
}
- public ResultSet getQueryResultAndWait(QueryId queryId)
+ public static ResultSet createResultSet(TajoClient client, QueryId queryId, GetQueryResultResponse response)
+ throws IOException {
+ TableDesc desc = CatalogUtil.newTableDesc(response.getTableDesc());
+ TajoConf conf = new TajoConf(client.getConf());
+ conf.setVar(ConfVars.USERNAME, response.getTajoUserName());
+ return new TajoResultSet(client, queryId, conf, desc);
+ }
+
+ public static ResultSet createResultSet(TajoClient client, SubmitQueryResponse response) throws IOException {
+ if (response.hasTableDesc()) {
+ TajoConf conf = new TajoConf(client.getConf());
+ conf.setVar(ConfVars.USERNAME, response.getUserName());
+ if (response.hasMaxRowNum()) {
+ return new TajoResultSet(
+ client,
+ QueryIdFactory.NULL_QUERY_ID,
+ conf,
+ new TableDesc(response.getTableDesc()),
+ response.getMaxRowNum());
+ } else {
+ return new TajoResultSet(
+ client,
+ QueryIdFactory.NULL_QUERY_ID,
+ conf,
+ new TableDesc(response.getTableDesc()));
+ }
+ } else {
+ SerializedResultSet serializedResultSet = response.getResultSet();
+ return new TajoMemoryResultSet(
+ new Schema(serializedResultSet.getSchema()),
+ serializedResultSet.getSerializedTuplesList(),
+ response.getMaxRowNum());
+ }
+ }
+
+ private ResultSet getQueryResultAndWait(QueryId queryId)
throws ServiceException, IOException {
if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
return createNullResultSet(queryId);
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java
new file mode 100644
index 0000000..84fafda
--- /dev/null
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoMemoryResultSet.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.jdbc;
+
+import com.google.protobuf.ByteString;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.storage.RowStoreUtil;
+import org.apache.tajo.storage.Tuple;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class TajoMemoryResultSet extends TajoResultSetBase {
+ private List<ByteString> serializedTuples;
+ private AtomicBoolean closed = new AtomicBoolean(false);
+ private RowStoreUtil.RowStoreDecoder decoder;
+
+ public TajoMemoryResultSet(Schema schema, List<ByteString> serializedTuples, int maxRowNum) {
+ this.schema = schema;
+ this.totalRow = maxRowNum;
+ this.serializedTuples = serializedTuples;
+ decoder = RowStoreUtil.createDecoder(schema);
+ init();
+ }
+
+ @Override
+ protected void init() {
+ cur = null;
+ curRow = 0;
+ }
+
+ @Override
+ public synchronized void close() throws SQLException {
+ if (closed.getAndSet(true)) {
+ return;
+ }
+
+ cur = null;
+ curRow = -1;
+ serializedTuples = null;
+ }
+
+ @Override
+ public void beforeFirst() throws SQLException {
+ curRow = 0;
+ }
+
+ @Override
+ protected Tuple nextTuple() throws IOException {
+ if (curRow < totalRow) {
+ cur = decoder.toTuple(serializedTuples.get(curRow).toByteArray());
+ return cur;
+ } else {
+ return null;
+ }
+ }
+
+ public boolean hasResult() {
+ return serializedTuples.size() > 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
index a64cea7..336c782 100644
--- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.tajo.QueryId;
+import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
@@ -42,11 +43,14 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class TajoResultSet extends TajoResultSetBase {
+ private static final int INFINITE_ROW_NUM = Integer.MAX_VALUE;
+
private FileSystem fs;
private Scanner scanner;
private TajoClient tajoClient;
private TajoConf conf;
private TableDesc desc;
+ private Long maxRowNum = null;
private QueryId queryId;
private AtomicBoolean closed = new AtomicBoolean(false);
@@ -56,26 +60,36 @@ public class TajoResultSet extends TajoResultSetBase {
init();
}
- public TajoResultSet(TajoClient tajoClient, QueryId queryId,
- TajoConf conf, TableDesc desc) throws IOException {
+ public TajoResultSet(TajoClient tajoClient, QueryId queryId, TajoConf conf, TableDesc table) throws IOException {
this.tajoClient = tajoClient;
this.queryId = queryId;
this.conf = conf;
- this.desc = desc;
+ this.desc = table;
+ initScanner();
+ init();
+ }
+ public TajoResultSet(TajoClient tajoClient, QueryId queryId, TajoConf conf, TableDesc table, long maxRowNum)
+ throws IOException {
+ this(tajoClient, queryId, conf, table);
+ this.maxRowNum = maxRowNum;
initScanner();
init();
}
private void initScanner() throws IOException {
if(desc != null) {
- this.schema = desc.getSchema();
-
+ schema = desc.getSchema();
fs = FileScanner.getFileSystem(conf, desc.getPath());
- this.totalRow = desc.getStats() != null ? desc.getStats().getNumRows() : 0;
+ if (maxRowNum != null) {
+ this.totalRow = maxRowNum;
+ } else {
+ this.totalRow = desc.getStats() != null ? desc.getStats().getNumRows() : INFINITE_ROW_NUM;
+ }
+
List<FileFragment> frags = getFragments(desc.getPath());
- scanner = new MergeScanner(conf, schema, desc.getMeta(), frags);
+ scanner = new MergeScanner(conf, desc.getSchema(), desc.getMeta(), frags);
}
}
@@ -128,7 +142,7 @@ public class TajoResultSet extends TajoResultSetBase {
}
try {
- if(tajoClient != null) {
+ if(tajoClient != null && !queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
this.tajoClient.closeQuery(queryId);
}
} catch (Exception e) {
@@ -165,6 +179,11 @@ public class TajoResultSet extends TajoResultSetBase {
if(scanner == null) {
return null;
}
+
+ if (maxRowNum != null && curRow >= maxRowNum) {
+ return null;
+ }
+
Tuple tuple = scanner.next();
if (tuple == null) {
//query is closed automatically by querymaster but scanner is not
@@ -182,4 +201,8 @@ public class TajoResultSet extends TajoResultSetBase {
public QueryId getQueryId() {
return queryId;
}
+
+ public TableDesc getTableDesc() {
+ return desc;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto
index 6e69a66..3baeee9 100644
--- a/tajo-client/src/main/proto/ClientProtos.proto
+++ b/tajo-client/src/main/proto/ClientProtos.proto
@@ -74,12 +74,6 @@ message UpdateQueryResponse {
optional string errorMessage = 2;
}
-message SubmitQueryResponse {
- required ResultCode resultCode = 1;
- optional QueryIdProto queryId = 2;
- optional string errorMessage = 3;
-}
-
message GetQueryResultRequest {
optional SessionIdProto sessionId = 1;
required QueryIdProto queryId = 2;
@@ -120,6 +114,28 @@ message GetQueryStatusRequest {
required QueryIdProto queryId = 2;
}
+message SubmitQueryResponse {
+ required ResultCode resultCode = 1;
+ required QueryIdProto queryId = 2;
+ required string userName = 3;
+ optional bool isForwarded = 4 [default = false];
+
+ optional string queryMasterHost = 5;
+ optional int32 queryMasterPort = 6;
+
+ message SerializedResultSet {
+ optional SchemaProto schema = 1;
+ optional int32 bytesNum = 2;
+ repeated bytes serializedTuples = 3;
+ }
+
+ optional SerializedResultSet resultSet = 7;
+ optional TableDescProto tableDesc = 8;
+ optional int32 maxRowNum = 9;
+
+ optional string errorMessage = 10;
+}
+
message GetQueryStatusResponse {
required ResultCode resultCode = 1;
required QueryIdProto queryId = 2;
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
index 3bfd9df..9495fb1 100644
--- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
+++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
@@ -39,8 +39,7 @@ service TajoMasterClientProtocolService {
rpc getAllSessionVariables(SessionIdProto) returns (KeyValueSetProto);
// Query Submission and Result APIs
- rpc explainQuery(SessionedStringProto) returns (ExplainQueryResponse);
- rpc submitQuery(QueryRequest) returns (GetQueryStatusResponse);
+ rpc submitQuery(QueryRequest) returns (SubmitQueryResponse);
rpc updateQuery(QueryRequest) returns (UpdateQueryResponse);
rpc getQueryResult(GetQueryResultRequest) returns (GetQueryResultResponse);
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
index 3b8f9cf..7fa7973 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
@@ -215,10 +215,10 @@ DROP : D R O P;
EPOCH : E P O C H;
EVERY : E V E R Y;
EXISTS : E X I S T S;
+EXPLAIN : E X P L A I N;
EXTERNAL : E X T E R N A L;
EXTRACT : E X T R A C T;
-
FILTER : F I L T E R;
FIRST : F I R S T;
FORMAT : F O R M A T;
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index f25b41f..1249d57 100644
--- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -35,7 +35,11 @@ options {
===============================================================================
*/
sql
- : statement (SEMI_COLON)? EOF
+ : (explain_clause)? statement (SEMI_COLON)? EOF
+ ;
+
+explain_clause
+ : EXPLAIN
;
statement
@@ -205,7 +209,9 @@ identifier
;
nonreserved_keywords
- : AVG
+ : ADD
+ | AVG
+ | ALTER
| BETWEEN
| BY
| CENTURY
@@ -224,6 +230,7 @@ nonreserved_keywords
| EPOCH
| EVERY
| EXISTS
+ | EXPLAIN
| EXTERNAL
| EXTRACT
| FILTER
@@ -259,6 +266,7 @@ nonreserved_keywords
| QUARTER
| RANGE
| REGEXP
+ | RENAME
| RLIKE
| ROLLUP
| SECOND
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 3edf768..e95d5af 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -71,7 +71,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public Expr visitSql(SqlContext ctx) {
- return visit(ctx.statement());
+ Expr statement = visit(ctx.statement());
+ if (checkIfExist(ctx.explain_clause())) {
+ return new Explain(statement);
+ } else {
+ return statement;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
index 0ea2c77..6aa4830 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java
@@ -39,6 +39,7 @@ public interface AlgebraVisitor<CONTEXT, RESULT> {
RESULT visitRelationList(CONTEXT ctx, Stack<Expr> stack, RelationList expr) throws PlanningException;
RESULT visitRelation(CONTEXT ctx, Stack<Expr> stack, Relation expr) throws PlanningException;
RESULT visitScalarSubQuery(CONTEXT ctx, Stack<Expr> stack, ScalarSubQuery expr) throws PlanningException;
+ RESULT visitExplain(CONTEXT ctx, Stack<Expr> stack, Explain expr) throws PlanningException;
// Data definition language
RESULT visitCreateDatabase(CONTEXT ctx, Stack<Expr> stack, CreateDatabase expr) throws PlanningException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
index 6f217a7..44adf12 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java
@@ -96,6 +96,9 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
case ScalarSubQuery:
current = visitScalarSubQuery(ctx, stack, (ScalarSubQuery) expr);
break;
+ case Explain:
+ current = visitExplain(ctx, stack, (Explain) expr);
+ break;
case CreateDatabase:
current = visitCreateDatabase(ctx, stack, (CreateDatabase) expr);
@@ -407,6 +410,14 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
return visitDefaultUnaryExpr(ctx, stack, expr);
}
+ @Override
+ public RESULT visitExplain(CONTEXT ctx, Stack<Expr> stack, Explain expr) throws PlanningException {
+ stack.push(expr);
+ RESULT child = visit(ctx, stack, expr.getChild());
+ stack.pop();
+ return child;
+ }
+
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// Data Definition Language Section
///////////////////////////////////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
index a24f25f..52dbde8 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java
@@ -66,6 +66,7 @@ public class LogicalPlan {
private List<String> planingHistory = Lists.newArrayList();
LogicalPlanner planner;
+ private boolean isExplain;
private final String currentDatabase;
public LogicalPlan(String currentDatabase, LogicalPlanner planner) {
@@ -104,6 +105,14 @@ public class LogicalPlan {
}
}
+ public void setExplain() {
+ isExplain = true;
+ }
+
+ public boolean isExplain() {
+ return isExplain;
+ }
+
/**
* Create a new {@link QueryBlock} and Get
*
@@ -313,7 +322,8 @@ public class LogicalPlan {
// The condition (currentNode.getInSchema().contains(column)) means
// the column can be used at the current node. So, we don't need to find aliase name.
- if (currentNode != null && !currentNode.getInSchema().contains(column) && currentNode.getType() != NodeType.TABLE_SUBQUERY) {
+ if (currentNode != null && !currentNode.getInSchema().contains(column)
+ && currentNode.getType() != NodeType.TABLE_SUBQUERY) {
List<Column> candidates = TUtil.newList();
if (block.namedExprsMgr.isAliased(qualifiedName)) {
String alias = block.namedExprsMgr.getAlias(canonicalName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index 317c05b..6336f50 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -165,6 +165,11 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return current;
}
+ public LogicalNode visitExplain(PlanContext ctx, Stack<Expr> stack, Explain expr) throws PlanningException {
+ ctx.plan.setExplain();
+ return visit(ctx, stack, expr.getChild());
+ }
+
/*===============================================================================================
Data Manupulation Language (DML) SECTION
===============================================================================================*/
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index ccd9847..68578e8 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -55,6 +55,65 @@ public class PlannerUtil {
}
/**
+ * Checks whether the query is simple or not.
+ * The simple query can be defined as 'select * from tb_name [LIMIT X]'.
+ *
+ * @param plan The logical plan
+ * @return True if the query is a simple query.
+ */
+ public static boolean checkIfSimpleQuery(LogicalPlan plan) {
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+
+ // one block, without where clause, no group-by, no-sort, no-join
+ boolean isOneQueryBlock = plan.getQueryBlocks().size() == 1;
+ boolean simpleOperator = rootNode.getChild().getType() == NodeType.LIMIT
+ || rootNode.getChild().getType() == NodeType.SCAN;
+ boolean noOrderBy = !plan.getRootBlock().hasNode(NodeType.SORT);
+ boolean noGroupBy = !plan.getRootBlock().hasNode(NodeType.GROUP_BY);
+ boolean noWhere = !plan.getRootBlock().hasNode(NodeType.SELECTION);
+ boolean noJoin = !plan.getRootBlock().hasNode(NodeType.JOIN);
+ boolean singleRelation = plan.getRootBlock().hasNode(NodeType.SCAN)
+ && PlannerUtil.getRelationLineage(plan.getRootBlock().getRoot()).length == 1;
+
+ boolean noComplexComputation = false;
+ if (singleRelation) {
+ ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
+ if (!scanNode.getTableDesc().hasPartition() && scanNode.hasTargets()
+ && scanNode.getTargets().length == scanNode.getInSchema().size()) {
+ noComplexComputation = true;
+ for (int i = 0; i < scanNode.getTargets().length; i++) {
+ noComplexComputation = noComplexComputation && scanNode.getTargets()[i].getEvalTree().getType() == EvalType.FIELD;
+ if (noComplexComputation) {
+ noComplexComputation = noComplexComputation && scanNode.getTargets()[i].getNamedColumn().equals(scanNode.getInSchema().getColumn(i));
+ }
+ if (!noComplexComputation) {
+ return noComplexComputation;
+ }
+ }
+ }
+ }
+
+ return !checkIfDDLPlan(rootNode) &&
+ (simpleOperator && noComplexComputation && isOneQueryBlock && noOrderBy && noGroupBy && noWhere && noJoin && singleRelation);
+ }
+
+ /**
+ * Checks whether the query has 'from clause' or not.
+ *
+ * @param plan The logical plan
+ * @return True if a query does not have 'from clause'.
+ */
+ public static boolean checkIfNonFromQuery(LogicalPlan plan) {
+ LogicalNode node = plan.getRootBlock().getRoot();
+
+ // one block, without where clause, no group-by, no-sort, no-join
+ boolean isOneQueryBlock = plan.getQueryBlocks().size() == 1;
+ boolean noRelation = !plan.getRootBlock().hasAlgebraicExpr(OpType.Relation);
+
+ return !checkIfDDLPlan(node) && noRelation && isOneQueryBlock;
+ }
+
+ /**
* Get all RelationNodes which are descendant of a given LogicalNode.
*
* @param from The LogicalNode to start visiting LogicalNodes.
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 9809aee..86f4935 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -30,6 +30,7 @@ import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.eval.EvalNode;
+import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleRange;
@@ -46,7 +47,7 @@ public class TupleUtil {
public static String rangeToQuery(Schema schema, TupleRange range, boolean last)
throws UnsupportedEncodingException {
- return rangeToQuery(range, last, RowStoreEncoder.createInstance(schema));
+ return rangeToQuery(range, last, RowStoreUtil.createEncoder(schema));
}
public static String rangeToQuery(TupleRange range, boolean last, RowStoreEncoder encoder)
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
index a56284b..cbb8f37 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -19,6 +19,7 @@
package org.apache.tajo.master;
import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -27,14 +28,16 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryIdFactory;
-import org.apache.tajo.TajoProtos;
import org.apache.tajo.algebra.Expr;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.engine.eval.EvalNode;
import org.apache.tajo.engine.exception.IllegalQueryStatusException;
import org.apache.tajo.engine.exception.VerifyException;
import org.apache.tajo.engine.parser.HiveQLAnalyzer;
@@ -47,8 +50,7 @@ import org.apache.tajo.master.TajoMaster.MasterContext;
import org.apache.tajo.master.querymaster.QueryInfo;
import org.apache.tajo.master.querymaster.QueryJobManager;
import org.apache.tajo.master.session.Session;
-import org.apache.tajo.storage.AbstractStorageManager;
-import org.apache.tajo.storage.StorageUtil;
+import org.apache.tajo.storage.*;
import java.io.IOException;
import java.sql.SQLException;
@@ -56,7 +58,9 @@ import java.util.ArrayList;
import java.util.List;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.apache.tajo.ipc.ClientProtos.GetQueryStatusResponse;
+
+import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse;
+import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse.SerializedResultSet;
public class GlobalEngine extends AbstractService {
/** Class Logger */
@@ -103,7 +107,7 @@ public class GlobalEngine extends AbstractService {
super.stop();
}
- public GetQueryStatusResponse executeQuery(Session session, String sql)
+ public SubmitQueryResponse executeQuery(Session session, String sql)
throws InterruptedException, IOException, IllegalQueryStatusException {
LOG.info("SQL: " + sql);
@@ -114,13 +118,13 @@ public class GlobalEngine extends AbstractService {
String [] cmds = sql.split(" ");
if(cmds != null) {
if(cmds[0].equalsIgnoreCase("set")) {
- String[] params = cmds[1].split("=");
- context.getConf().set(params[0], params[1]);
- GetQueryStatusResponse.Builder responseBuilder = GetQueryStatusResponse.newBuilder();
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- responseBuilder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED);
- return responseBuilder.build();
+ String[] params = cmds[1].split("=");
+ context.getConf().set(params[0], params[1]);
+ SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
+ responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ return responseBuilder.build();
}
}
@@ -136,49 +140,16 @@ public class GlobalEngine extends AbstractService {
Expr planningContext = hiveQueryMode ? converter.parse(sql) : analyzer.parse(sql);
LogicalPlan plan = createLogicalPlan(session, planningContext);
- LogicalRootNode rootNode = plan.getRootBlock().getRoot();
-
- GetQueryStatusResponse.Builder responseBuilder = GetQueryStatusResponse.newBuilder();
- if (PlannerUtil.checkIfDDLPlan(rootNode)) {
- context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- updateQuery(session, rootNode.getChild());
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- responseBuilder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED);
- } else {
- context.getSystemMetrics().counter("Query", "numDMLQuery").inc();
- hookManager.doHooks(queryContext, plan);
-
- QueryJobManager queryJobManager = this.context.getQueryJobManager();
- QueryInfo queryInfo;
-
- queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, rootNode);
-
- if(queryInfo == null) {
- responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
- responseBuilder.setState(TajoProtos.QueryState.QUERY_ERROR);
- responseBuilder.setErrorMessage("Fail starting QueryMaster.");
- } else {
- responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- responseBuilder.setState(queryInfo.getQueryState());
- if(queryInfo.getQueryMasterHost() != null) {
- responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
- }
- responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
- }
- }
- GetQueryStatusResponse response = responseBuilder.build();
-
+ SubmitQueryResponse response = executeQueryInternal(queryContext, session, plan, sql);
return response;
} catch (Throwable t) {
context.getSystemMetrics().counter("Query", "errorQuery").inc();
LOG.error("\nStack Trace:\n" + StringUtils.stringifyException(t));
- GetQueryStatusResponse.Builder responseBuilder = GetQueryStatusResponse.newBuilder();
+ SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
+ responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setIsForwarded(true);
responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
- responseBuilder.setState(TajoProtos.QueryState.QUERY_ERROR);
String errorMessage = t.getMessage();
if (t.getMessage() == null) {
errorMessage = StringUtils.stringifyException(t);
@@ -188,16 +159,115 @@ public class GlobalEngine extends AbstractService {
}
}
- public String explainQuery(Session session, String sql) throws IOException, SQLException, PlanningException {
- LOG.info("SQL: " + sql);
- // parse the query
+ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext,
+ Session session,
+ LogicalPlan plan,
+ String sql) throws Exception {
+
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+
+ SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder();
+ responseBuilder.setIsForwarded(false);
+ responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME));
+
+ if (PlannerUtil.checkIfDDLPlan(rootNode)) {
+ context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
+ updateQuery(session, rootNode.getChild());
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+
+ } else if (plan.isExplain()) { // explain query
+ String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
+ Schema schema = new Schema();
+ schema.addColumn("explain", TajoDataTypes.Type.TEXT);
+ RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
+
+ SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();
+
+ VTuple tuple = new VTuple(1);
+ String[] lines = explainStr.split("\n");
+ int bytesNum = 0;
+ for (String line : lines) {
+ tuple.put(0, DatumFactory.createText(line));
+ byte [] encodedData = encoder.toBytes(tuple);
+ bytesNum += encodedData.length;
+ serializedResBuilder.addSerializedTuples(ByteString.copyFrom(encodedData));
+ }
+ serializedResBuilder.setSchema(schema.getProto());
+ serializedResBuilder.setBytesNum(bytesNum);
+
+ responseBuilder.setResultSet(serializedResBuilder.build());
+ responseBuilder.setMaxRowNum(lines.length);
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- final boolean hiveQueryMode = context.getConf().getBoolVar(TajoConf.ConfVars.HIVE_QUERY_MODE);
- Expr planningContext = hiveQueryMode ? converter.parse(sql) : analyzer.parse(sql);
- LOG.info("hive.query.mode:" + hiveQueryMode);
+ // Simple query indicates a form of 'select * from tb_name [LIMIT X];'.
+ } else if (PlannerUtil.checkIfSimpleQuery(plan)) {
+ ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
+ TableDesc desc = scanNode.getTableDesc();
+ if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
+ LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
+ responseBuilder.setMaxRowNum((int) limitNode.getFetchFirstNum());
+ } else {
+ if (desc.getStats().getNumBytes() > 0 && desc.getStats().getNumRows() == 0) {
+ responseBuilder.setMaxRowNum(Integer.MAX_VALUE);
+ }
+ }
+ responseBuilder.setTableDesc(desc.getProto());
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+
+ // NonFromQuery indicates a form of 'select a, x+y;'
+ } else if (PlannerUtil.checkIfNonFromQuery(plan)) {
+ Target [] targets = plan.getRootBlock().getRawTargets();
+ if (targets == null) {
+ throw new PlanningException("No targets");
+ }
+ Tuple outTuple = new VTuple(targets.length);
+ for (int i = 0; i < targets.length; i++) {
+ EvalNode eval = targets[i].getEvalTree();
+ outTuple.put(i, eval.eval(null, null));
+ }
+
+ Schema schema = PlannerUtil.targetToSchema(targets);
+ RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
+ byte [] serializedBytes = encoder.toBytes(outTuple);
+ SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();
+ serializedResBuilder.addSerializedTuples(ByteString.copyFrom(serializedBytes));
+ serializedResBuilder.setSchema(schema.getProto());
+ serializedResBuilder.setBytesNum(serializedBytes.length);
+
+ responseBuilder.setResultSet(serializedResBuilder);
+ responseBuilder.setMaxRowNum(1);
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
- LogicalPlan plan = createLogicalPlan(session, planningContext);
- return plan.toString();
+ } else { // it requires distributed execution. So, the query is forwarded to a query master.
+ context.getSystemMetrics().counter("Query", "numDMLQuery").inc();
+ hookManager.doHooks(queryContext, plan);
+
+ QueryJobManager queryJobManager = this.context.getQueryJobManager();
+ QueryInfo queryInfo;
+
+ queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, rootNode);
+
+ if(queryInfo == null) {
+ responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
+ responseBuilder.setErrorMessage("Fail starting QueryMaster.");
+ } else {
+ responseBuilder.setIsForwarded(true);
+ responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
+ responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ if(queryInfo.getQueryMasterHost() != null) {
+ responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
+ }
+ responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
+ }
+ LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort());
+ }
+ SubmitQueryResponse response = responseBuilder.build();
+ return response;
}
public QueryId updateQuery(Session session, String sql) throws IOException, SQLException, PlanningException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index eed1007..c968a73 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -248,33 +248,7 @@ public class TajoMasterClientService extends AbstractService {
}
@Override
- public ExplainQueryResponse explainQuery(RpcController controller,
- SessionedStringProto request) throws ServiceException {
-
- try {
- Session session = context.getSessionManager().getSession(request.getSessionId().getId());
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("ExplainQuery [" + request.getValue() + "]");
- }
- ClientProtos.ExplainQueryResponse.Builder responseBuilder = ClientProtos.ExplainQueryResponse.newBuilder();
- responseBuilder.setResultCode(ResultCode.OK);
- String plan = context.getGlobalEngine().explainQuery(session, request.getValue());
- if(LOG.isDebugEnabled()) {
- LOG.debug("ExplainQuery [" + plan + "]");
- }
- responseBuilder.setExplain(plan);
- return responseBuilder.build();
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- ClientProtos.ExplainQueryResponse.Builder responseBuilder = ClientProtos.ExplainQueryResponse.newBuilder();
- responseBuilder.setResultCode(ResultCode.ERROR);
- responseBuilder.setErrorMessage(e.getMessage());
- return responseBuilder.build();
- }
- }
- @Override
- public GetQueryStatusResponse submitQuery(RpcController controller, QueryRequest request) throws ServiceException {
+ public SubmitQueryResponse submitQuery(RpcController controller, QueryRequest request) throws ServiceException {
try {
@@ -286,7 +260,8 @@ public class TajoMasterClientService extends AbstractService {
return context.getGlobalEngine().executeQuery(session, request.getQuery());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
- ClientProtos.GetQueryStatusResponse.Builder responseBuilder = ClientProtos.GetQueryStatusResponse.newBuilder();
+ SubmitQueryResponse.Builder responseBuilder = ClientProtos.SubmitQueryResponse.newBuilder();
+ responseBuilder.setUserName(context.getConf().getVar(ConfVars.USERNAME));
responseBuilder.setResultCode(ResultCode.ERROR);
if (e.getMessage() != null) {
responseBuilder.setErrorMessage(ExceptionUtils.getStackTrace(e));
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index df8b31b..a72c222 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -410,7 +410,7 @@ public class Repartitioner {
Set<URI> uris;
try {
- RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.RowStoreEncoder.createInstance(sortSchema);
+ RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(sortSchema);
for (int i = 0; i < ranges.length; i++) {
uris = new HashSet<URI>();
for (String uri: basicFetchURIs) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
index 24eea83..faeadaf 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
@@ -221,7 +221,7 @@ public class QueryExecutorServlet extends HttpServlet {
String queryRunnerId;
- ClientProtos.GetQueryStatusResponse queryRespons;
+ ClientProtos.SubmitQueryResponse queryRespons;
AtomicBoolean running = new AtomicBoolean(true);
AtomicBoolean stop = new AtomicBoolean(false);
QueryId queryId;
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/RangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/RangeRetrieverHandler.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/RangeRetrieverHandler.java
index 0e8ae72..be33a12 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/RangeRetrieverHandler.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/RangeRetrieverHandler.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
@@ -71,7 +72,7 @@ public class RangeRetrieverHandler implements RetrieverHandler {
this.idxReader.open();
LOG.info("BSTIndex is loaded from disk (" + idxReader.getFirstKey() + ", "
+ idxReader.getLastKey());
- this.decoder = RowStoreDecoder.createInstance(schema);
+ this.decoder = RowStoreUtil.createDecoder(schema);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index e3ca39b..ed943f2 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -433,11 +433,11 @@ public class QueryTestCaseBase {
if (expr.getType() == OpType.CreateTable) {
CreateTable createTable = (CreateTable) expr;
- String tableName = createTable.getTableName();
+ String tableName = CatalogUtil.denormalizeIdentifier(createTable.getTableName());
assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getStatement()));
TableDesc createdTable = client.getTableDesc(tableName);
- String createdTableName = createdTable.getName();
+ String createdTableName = CatalogUtil.denormalizeIdentifier(createdTable.getName());
assertTrue("table '" + createdTableName + "' creation check", client.existTable(createdTableName));
if (isLocalTable) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/cli/TestSimpleParser.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
index e524652..9c02b65 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
@@ -28,6 +28,21 @@ import static org.junit.Assert.assertTrue;
public class TestSimpleParser {
@Test
+ public final void testSpecialCases() throws InvalidStatementException {
+ List<ParsedResult> res1 = SimpleParser.parseScript("");
+ assertEquals(0, res1.size());
+
+ List<ParsedResult> res2 = SimpleParser.parseScript("a");
+ assertEquals(1, res2.size());
+
+ List<ParsedResult> res3 = SimpleParser.parseScript("?");
+ assertEquals(0, res3.size());
+
+ List<ParsedResult> res4 = SimpleParser.parseScript("\\");
+ assertEquals(1, res4.size());
+ }
+
+ @Test
public final void testMetaCommands() throws InvalidStatementException {
List<ParsedResult> res1 = SimpleParser.parseScript("\\d");
assertEquals(1, res1.size());
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
index a2e3181..49e6874 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -198,7 +198,7 @@ public class TestTajoClient {
@Test
public final void testKillQuery() throws IOException, ServiceException, InterruptedException {
- ClientProtos.GetQueryStatusResponse res = client.executeQuery("select sleep(1) from lineitem");
+ ClientProtos.SubmitQueryResponse res = client.executeQuery("select sleep(1) from lineitem");
Thread.sleep(1000);
QueryId queryId = new QueryId(res.getQueryId());
client.killQuery(queryId);
@@ -600,10 +600,10 @@ public class TestTajoClient {
assertTrue(client.existTable(tableName));
int numFinishedQueries = client.getFinishedQueryList().size();
- ResultSet resultSet = client.executeQueryAndGetResult("select * from " + tableName);
+ ResultSet resultSet = client.executeQueryAndGetResult("select * from " + tableName + " order by deptname");
assertNotNull(resultSet);
- resultSet = client.executeQueryAndGetResult("select * from " + tableName);
+ resultSet = client.executeQueryAndGetResult("select * from " + tableName + " order by deptname");
assertNotNull(resultSet);
assertEquals(numFinishedQueries + 2, client.getFinishedQueryList().size());
@@ -616,8 +616,8 @@ public class TestTajoClient {
*/
@Test(timeout = 20 * 1000)
public final void testGetQueryStatusAndResultAfterFinish() throws Exception {
- String sql = "select * from lineitem";
- ClientProtos.GetQueryStatusResponse response = client.executeQuery(sql);
+ String sql = "select * from lineitem order by l_orderkey";
+ ClientProtos.SubmitQueryResponse response = client.executeQuery(sql);
assertNotNull(response);
QueryId queryId = new QueryId(response.getQueryId());
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index cf17d89..cee0cb0 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -896,7 +896,7 @@ public class TestPhysicalPlanner {
// The below is for testing RangeRetrieverHandler.
- RowStoreEncoder encoder = RowStoreEncoder.createInstance(keySchema);
+ RowStoreEncoder encoder = RowStoreUtil.createEncoder(keySchema);
RangeRetrieverHandler handler = new RangeRetrieverHandler(
new File(new Path(workDir, "output").toUri()), keySchema, comp);
Map<String,List<String>> kvs = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
index a75631a..96f3bbe 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java
@@ -41,6 +41,38 @@ public class TestSelectQuery extends QueryTestCaseBase {
}
@Test
+ public final void testNonFromSelect1() throws Exception {
+ // select upper('abc');
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSimpleQuery() throws Exception {
+ // select * from lineitem;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSimpleQueryWithLimit() throws Exception {
+ // select * from lineitem limit 3;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testExplainSelect() throws Exception {
+ // explain select l_orderkey, l_partkey from lineitem;
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
public final void testSelect() throws Exception {
// select l_orderkey, l_partkey from lineitem;
ResultSet res = executeQuery();
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java
index 86fa798..cecb281 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java
@@ -28,12 +28,9 @@ import org.apache.tajo.engine.planner.PlannerUtil;
import org.apache.tajo.engine.planner.RangePartitionAlgorithm;
import org.apache.tajo.engine.planner.UniformRangePartition;
import org.apache.tajo.engine.utils.TupleUtil;
+import org.apache.tajo.storage.*;
import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder;
-import org.apache.tajo.storage.Tuple;
-import org.apache.tajo.storage.TupleComparator;
-import org.apache.tajo.storage.TupleRange;
-import org.apache.tajo.storage.VTuple;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -70,8 +67,8 @@ public class TestTupleUtil {
DatumFactory.createInet4("192.168.0.1")
});
- RowStoreEncoder encoder = RowStoreEncoder.createInstance(schema);
- RowStoreDecoder decoder = RowStoreDecoder.createInstance(schema);
+ RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
+ RowStoreDecoder decoder = RowStoreUtil.createDecoder(schema);
byte [] bytes = encoder.toBytes(tuple);
Tuple tuple2 = decoder.toTuple(bytes);
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index 4e770ce..686aa9c 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -356,7 +356,7 @@ public class TestRangeRetrieverHandler {
private FileChunk getFileChunk(RangeRetrieverHandler handler, Schema keySchema,
TupleRange range, boolean last) throws IOException {
Map<String,List<String>> kvs = Maps.newHashMap();
- RowStoreEncoder encoder = RowStoreEncoder.createInstance(keySchema);
+ RowStoreEncoder encoder = RowStoreUtil.createEncoder(keySchema);
kvs.put("start", Lists.newArrayList(
new String(Base64.encodeBase64(
encoder.toBytes(range.getStart()),
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testExplainSelect.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testExplainSelect.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testExplainSelect.sql
new file mode 100644
index 0000000..4d4f32c
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testExplainSelect.sql
@@ -0,0 +1 @@
+explain select l_orderkey, l_partkey from lineitem;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testNonFromSelect1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testNonFromSelect1.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testNonFromSelect1.sql
new file mode 100644
index 0000000..ca0ef7a
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testNonFromSelect1.sql
@@ -0,0 +1 @@
+select upper('abc');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQuery.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQuery.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQuery.sql
new file mode 100644
index 0000000..cda515b
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQuery.sql
@@ -0,0 +1 @@
+select * from lineitem;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQueryWithLimit.sql
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQueryWithLimit.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQueryWithLimit.sql
new file mode 100644
index 0000000..79cf804
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestSelectQuery/testSimpleQueryWithLimit.sql
@@ -0,0 +1 @@
+select * from lineitem limit 3;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testExplainSelect.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testExplainSelect.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testExplainSelect.result
new file mode 100644
index 0000000..a6aa8f4
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testExplainSelect.result
@@ -0,0 +1,6 @@
+explain
+-------------------------------
+SCAN(0) on default.lineitem
+ => target list: default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4)
+ => out schema: {(2) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4)}
+ => in schema: {(16) default.lineitem.l_orderkey (INT4),default.lineitem.l_partkey (INT4),default.lineitem.l_suppkey (INT4),default.lineitem.l_linenumber (INT4),default.lineitem.l_quantity (FLOAT8),default.lineitem.l_extendedprice (FLOAT8),default.lineitem.l_discount (FLOAT8),default.lineitem.l_tax (FLOAT8),default.lineitem.l_returnflag (TEXT),default.lineitem.l_linestatus (TEXT),default.lineitem.l_shipdate (TEXT),default.lineitem.l_commitdate (TEXT),default.lineitem.l_receiptdate (TEXT),default.lineitem.l_shipinstruct (TEXT),default.lineitem.l_shipmode (TEXT),default.lineitem.l_comment (TEXT)}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testNonFromSelect1.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testNonFromSelect1.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testNonFromSelect1.result
new file mode 100644
index 0000000..3d2155f
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testNonFromSelect1.result
@@ -0,0 +1,3 @@
+?upper
+-------------------------------
+ABC
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQuery.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQuery.result
new file mode 100644
index 0000000..ec6b911
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQuery.result
@@ -0,0 +1,7 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
+3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco
+3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL, unusual accounts. eve
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQueryWithLimit.result
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQueryWithLimit.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQueryWithLimit.result
new file mode 100644
index 0000000..ed89068
--- /dev/null
+++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestSelectQuery/testSimpleQueryWithLimit.result
@@ -0,0 +1,5 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
index afdae23..40978fe 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/PullServerAuxService.java
@@ -51,6 +51,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.pullserver.listener.FileCloseListener;
import org.apache.tajo.pullserver.retriever.FileChunk;
+import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
@@ -559,7 +560,7 @@ public class PullServerAuxService extends AuxiliaryService {
byte [] startBytes = Base64.decodeBase64(startKey);
byte [] endBytes = Base64.decodeBase64(endKey);
- RowStoreDecoder decoder = RowStoreDecoder.createInstance(keySchema);
+ RowStoreDecoder decoder = RowStoreUtil.createDecoder(keySchema);
Tuple start;
Tuple end;
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/631f3d04/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
index c416b66..70f5421 100644
--- a/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
+++ b/tajo-core/tajo-core-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java
@@ -48,6 +48,7 @@ import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.pullserver.listener.FileCloseListener;
import org.apache.tajo.pullserver.retriever.FileChunk;
import org.apache.tajo.rpc.RpcChannelFactory;
+import org.apache.tajo.storage.RowStoreUtil;
import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.TupleComparator;
@@ -556,7 +557,7 @@ public class TajoPullServerService extends AbstractService {
byte [] startBytes = Base64.decodeBase64(startKey);
byte [] endBytes = Base64.decodeBase64(endKey);
- RowStoreDecoder decoder = RowStoreDecoder.createInstance(keySchema);
+ RowStoreDecoder decoder = RowStoreUtil.createDecoder(keySchema);
Tuple start;
Tuple end;
try {