You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/03/24 09:39:53 UTC
[4/4] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/12846cca
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/12846cca
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/12846cca
Branch: refs/heads/index_support
Commit: 12846ccae5bd5c3673d8812d53010ef432c947c6
Parents: d09bd8d 3e9a2dd
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Mar 24 17:39:46 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Mar 24 17:39:46 2015 +0900
----------------------------------------------------------------------
CHANGES | 9 +
.../java/org/apache/tajo/algebra/Explain.java | 9 +-
.../main/java/org/apache/tajo/SessionVars.java | 1 +
.../java/org/apache/tajo/conf/TajoConf.java | 1 +
.../org/apache/tajo/datum/BooleanDatum.java | 9 +-
.../org/apache/tajo/datum/DatumFactory.java | 4 +
.../java/org/apache/tajo/datum/TextDatum.java | 4 +-
.../java/org/apache/tajo/util/FileUtil.java | 25 ++-
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 1 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 2 +-
.../engine/planner/global/GlobalPlanner.java | 4 +
.../engine/planner/physical/MemSortExec.java | 28 ++-
.../tajo/engine/planner/physical/SortExec.java | 4 +-
.../engine/planner/physical/TupleSorter.java | 27 +++
.../planner/physical/VectorizedSorter.java | 198 +++++++++++++++++
.../org/apache/tajo/master/QueryInProgress.java | 22 +-
.../org/apache/tajo/master/QueryManager.java | 8 +-
.../ExplainGlobalPlanPreprocessorForTest.java | 62 ++++++
.../exec/ExplainPlanPreprocessorForTest.java | 218 +++++++++++++++++++
.../apache/tajo/master/exec/QueryExecutor.java | 61 +++++-
.../tajo/querymaster/QueryMasterTask.java | 3 -
.../java/org/apache/tajo/QueryTestCaseBase.java | 75 ++++++-
.../planner/physical/TestTupleSorter.java | 132 +++++++++++
.../tajo/engine/query/TestSelectQuery.java | 16 ++
.../testExplainSelectPhysical.1.result | 26 +++
.../testExplainSelectPhysical.2.result | 88 ++++++++
.../testExplainSelectPhysical.3.result | 89 ++++++++
.../java/org/apache/tajo/plan/LogicalPlan.java | 17 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 2 +-
.../apache/tajo/plan/logical/IndexScanNode.java | 4 +-
31 files changed, 1093 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index 31a28f7,75e7762..ca3abbd
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@@ -48,6 -47,7 +50,8 @@@ import org.apache.tajo.master.*
import org.apache.tajo.master.exec.prehook.CreateTableHook;
import org.apache.tajo.master.exec.prehook.DistributedQueryHookManager;
import org.apache.tajo.master.exec.prehook.InsertIntoHook;
+ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
++import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.querymaster.*;
import org.apache.tajo.session.Session;
import org.apache.tajo.plan.LogicalPlan;
@@@ -99,20 -98,13 +103,20 @@@ public class QueryExecutor
} else if (PlannerUtil.checkIfDDLPlan(rootNode)) {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- ddlExecutor.execute(queryContext, plan);
- response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ if (PlannerUtil.isDistExecDDL(rootNode)) {
+ if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) {
+ checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild());
+ }
+ executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
+ } else {
+ response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ response.setResult(IPCUtil.buildOkRequestResult());
+ ddlExecutor.execute(queryContext, plan);
+ }
} else if (plan.isExplain()) { // explain query
- execExplain(plan, response);
+ execExplain(plan, queryContext, plan.isExplainGlobal(), response);
} else if (PlannerUtil.checkIfQueryTargetIsVirtualTable(plan)) {
execQueryOnVirtualTable(queryContext, session, sql, plan, response);
@@@ -165,12 -157,31 +169,31 @@@
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ response.setResult(IPCUtil.buildOkRequestResult());
}
- public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) throws IOException {
+ public void execExplain(LogicalPlan plan, QueryContext queryContext, boolean isGlobal,
+ SubmitQueryResponse.Builder response)
+ throws Exception {
+ String explainStr;
+ boolean isTest = queryContext.getBool(SessionVars.TEST_PLAN_SHAPE_FIX_ENABLED);
+ if (isTest) {
+ ExplainPlanPreprocessorForTest preprocessorForTest = new ExplainPlanPreprocessorForTest();
+ preprocessorForTest.prepareTest(plan);
+ }
+
+ if (isGlobal) {
+ GlobalPlanner planner = new GlobalPlanner(context.getConf(), context.getCatalog());
+ MasterPlan masterPlan = compileMasterPlan(plan, queryContext, planner);
+ if (isTest) {
+ ExplainGlobalPlanPreprocessorForTest globalPlanPreprocessorForTest = new ExplainGlobalPlanPreprocessorForTest();
+ globalPlanPreprocessorForTest.prepareTest(masterPlan);
+ }
+ explainStr = masterPlan.toString();
+ } else {
+ explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
+ }
- String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
Schema schema = new Schema();
schema.addColumn("explain", TajoDataTypes.Type.TEXT);
RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
@@@ -428,22 -439,33 +451,52 @@@
}
}
+ private void checkIndexExistence(final QueryContext queryContext, final CreateIndexNode createIndexNode)
+ throws IOException {
+ String databaseName, simpleIndexName, qualifiedIndexName;
+ if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) {
- String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
++ String[] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
+ databaseName = splits[0];
+ simpleIndexName = splits[1];
+ qualifiedIndexName = createIndexNode.getIndexName();
+ } else {
+ databaseName = queryContext.getCurrentDatabase();
+ simpleIndexName = createIndexNode.getIndexName();
+ qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName);
+ }
+
+ if (catalog.existIndexByName(databaseName, simpleIndexName)) {
+ throw new AlreadyExistsIndexException(qualifiedIndexName);
+ }
+ }
++
+ public static MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
+ throws Exception {
+
+ CatalogProtos.StoreType storeType = PlannerUtil.getStoreType(plan);
+ if (storeType != null) {
+ StorageManager sm = StorageManager.getStorageManager(planner.getConf(), storeType);
+ StorageProperty storageProperty = sm.getStorageProperty();
+ if (storageProperty.isSortedInsert()) {
+ String tableName = PlannerUtil.getStoreTableName(plan);
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+ TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());
+ if (tableDesc == null) {
+ throw new VerifyException("Can't get table meta data from catalog: " + tableName);
+ }
+ List<LogicalPlanRewriteRule> storageSpecifiedRewriteRules = sm.getRewriteRules(
+ context, tableDesc);
+ if (storageSpecifiedRewriteRules != null) {
+ for (LogicalPlanRewriteRule eachRule: storageSpecifiedRewriteRules) {
- eachRule.rewrite(context, plan);
++ eachRule.rewrite(new LogicalPlanRewriteRuleContext(context, plan));
+ }
+ }
+ }
+ }
+
+ MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
+ planner.build(masterPlan);
+
+ return masterPlan;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
index d38e2da,f83cb1e..26111c3
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java
@@@ -51,12 -46,15 +51,9 @@@ import org.apache.tajo.ipc.TajoWorkerPr
import org.apache.tajo.master.TajoContainerProxy;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.rm.TajoWorkerResourceManager;
--import org.apache.tajo.plan.LogicalOptimizer;
--import org.apache.tajo.plan.LogicalPlan;
--import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.plan.logical.LogicalRootNode;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.session.Session;
import org.apache.tajo.storage.StorageManager;
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/12846cca/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
----------------------------------------------------------------------
diff --cc tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
index e320ce9,8b73756..ebf6c4b
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
@@@ -82,9 -84,9 +82,9 @@@ public class IndexScanNode extends Scan
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
-- result = prime * result + Arrays.hashCode(datum);
++ result = prime * result + indexPath.hashCode();
result = prime * result + ((keySchema == null) ? 0 : keySchema.hashCode());
-- result = prime * result + Arrays.hashCode(sortKeys);
++ result = prime * result + Arrays.hashCode(predicates);
return result;
}