You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/01/08 17:14:48 UTC
[1/5] tajo git commit: TAJO-838: Improve query planner to utilize
index. (jihoon)
Repository: tajo
Updated Branches:
refs/heads/index_support 1fad72ebe -> 071c5d05d
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index d7cd82e..1ce7019 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -23,14 +23,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.algebra.JoinType;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.*;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
+import org.apache.tajo.plan.util.IndexUtil;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
@@ -47,6 +48,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
private final static Log LOG = LogFactory.getLog(FilterPushDownRule.class);
private static final String NAME = "FilterPushDown";
+ private CatalogService catalog;
+
static class FilterPushDownContext {
Set<EvalNode> pushingDownFilters = new HashSet<EvalNode>();
@@ -80,8 +83,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) {
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
+ for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) {
if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) {
return true;
}
@@ -90,7 +93,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws PlanningException {
/*
FilterPushDown rule: processing when visits each node
- If a target which is corresponding on a filter EvalNode's column is not FieldEval, do not PushDown.
@@ -102,6 +105,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
. It not, create new HavingNode and set parent's child.
*/
FilterPushDownContext context = new FilterPushDownContext();
+ LogicalPlan plan = rewriteRuleContext.getPlan();
+ catalog = rewriteRuleContext.getCatalog();
for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) {
context.clear();
this.visit(context, plan, block, block.getRoot(), new Stack<LogicalNode>());
@@ -834,7 +839,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
@Override
public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan,
- LogicalPlan.QueryBlock block, ScanNode scanNode,
+ LogicalPlan.QueryBlock block, final ScanNode scanNode,
Stack<LogicalNode> stack) throws PlanningException {
List<EvalNode> matched = Lists.newArrayList();
@@ -904,8 +909,42 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
qual = matched.iterator().next();
}
+ block.addAccessPath(scanNode, new SeqScanInfo(table));
if (qual != null) { // if a matched qual exists
scanNode.setQual(qual);
+
+ // Index path can be identified only after filters are pushed into each scan.
+ String databaseName, tableName;
+ databaseName = CatalogUtil.extractQualifier(table.getName());
+ tableName = CatalogUtil.extractSimpleName(table.getName());
+ Set<Predicate> predicates = TUtil.newHashSet();
+ for (EvalNode eval : IndexUtil.getAllEqualEvals(qual)) {
+ BinaryEval binaryEval = (BinaryEval) eval;
+ // TODO: consider more complex predicates
+ if (binaryEval.getLeftExpr().getType() == EvalType.FIELD &&
+ binaryEval.getRightExpr().getType() == EvalType.CONST) {
+ predicates.add(new Predicate(binaryEval.getType(),
+ ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(),
+ ((ConstEval)binaryEval.getRightExpr()).getValue()));
+ } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST &&
+ binaryEval.getRightExpr().getType() == EvalType.FIELD) {
+ predicates.add(new Predicate(binaryEval.getType(),
+ ((FieldEval) binaryEval.getRightExpr()).getColumnRef(),
+ ((ConstEval)binaryEval.getLeftExpr()).getValue()));
+ }
+ }
+
+ // for every subset of the set of columns, find all matched index paths
+ for (Set<Predicate> subset : Sets.powerSet(predicates)) {
+ if (subset.size() == 0)
+ continue;
+ Column[] columns = extractColumns(subset);
+ if (catalog.existIndexByColumns(databaseName, tableName, columns)) {
+ IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns);
+ block.addAccessPath(scanNode, new IndexScanInfo(
+ table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset)));
+ }
+ }
}
for (EvalNode matchedEval: matched) {
@@ -918,6 +957,49 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
return scanNode;
}
+ private static class Predicate {
+ Column column;
+ Datum value;
+ EvalType evalType;
+
+ public Predicate(EvalType evalType, Column column, Datum value) {
+ this.evalType = evalType;
+ this.column = column;
+ this.value = value;
+ }
+ }
+
+ private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set<Predicate> predicates) {
+ SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()];
+ Map<Column, Datum> colToValue = TUtil.newHashMap();
+ for (Predicate predicate : predicates) {
+ colToValue.put(predicate.column, predicate.value);
+ }
+ SortSpec [] keySortSpecs = desc.getKeySortSpecs();
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ simplePredicates[i] = new SimplePredicate(keySortSpecs[i],
+ colToValue.get(keySortSpecs[i].getSortKey()));
+ }
+ return simplePredicates;
+ }
+
+ private static Datum[] extractPredicateValues(List<Predicate> predicates) {
+ Datum[] values = new Datum[predicates.size()];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = predicates.get(i).value;
+ }
+ return values;
+ }
+
+ private static Column[] extractColumns(Set<Predicate> predicates) {
+ Column[] columns = new Column[predicates.size()];
+ int i = 0;
+ for (Predicate p : predicates) {
+ columns[i++] = p.column;
+ }
+ return columns;
+ }
+
private void errorFilterPushDown(LogicalPlan plan, LogicalNode node,
FilterPushDownContext context) throws PlanningException {
String notMatchedNodeStr = "";
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java
new file mode 100644
index 0000000..9ac8ccf
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite.rules;
+
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.IndexDesc;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SortSpec;
+import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.plan.serder.EvalNodeDeserializer;
+import org.apache.tajo.plan.serder.EvalNodeSerializer;
+import org.apache.tajo.plan.serder.PlanProto.SimplePredicateProto;
+
+import java.net.URI;
+
+public class IndexScanInfo extends AccessPathInfo {
+
+ /**
+ * Simple predicate represents an equal eval expression which consists of
+ * a column and a value.
+ */
+ // TODO: extend to represent more complex expressions
+ public static class SimplePredicate implements ProtoObject<SimplePredicateProto> {
+ @Expose private SortSpec keySortSpec;
+ @Expose private Datum value;
+
+ public SimplePredicate(SortSpec keySortSpec, Datum value) {
+ this.keySortSpec = keySortSpec;
+ this.value = value;
+ }
+
+ public SimplePredicate(SimplePredicateProto proto) {
+ keySortSpec = new SortSpec(proto.getKeySortSpec());
+ value = EvalNodeDeserializer.deserialize(proto.getValue());
+ }
+
+ public SortSpec getKeySortSpec() {
+ return keySortSpec;
+ }
+
+ public Datum getValue() {
+ return value;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof SimplePredicate) {
+ SimplePredicate other = (SimplePredicate) o;
+ return this.keySortSpec.equals(other.keySortSpec) && this.value.equals(other.value);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ SimplePredicate clone = new SimplePredicate(this.keySortSpec, this.value);
+ return clone;
+ }
+
+ @Override
+ public SimplePredicateProto getProto() {
+ SimplePredicateProto.Builder builder = SimplePredicateProto.newBuilder();
+ builder.setKeySortSpec(keySortSpec.getProto());
+ builder.setValue(EvalNodeSerializer.serialize(value));
+ return builder.build();
+ }
+ }
+
+ private final URI indexPath;
+ private final Schema keySchema;
+ private final SimplePredicate[] predicates;
+
+ public IndexScanInfo(TableStats tableStats, IndexDesc indexDesc, SimplePredicate[] predicates) {
+ super(ScanTypeControl.INDEX_SCAN, tableStats);
+ this.indexPath = indexDesc.getIndexPath();
+ keySchema = new Schema();
+ this.predicates = predicates;
+ for (SimplePredicate predicate : predicates) {
+ keySchema.addColumn(predicate.getKeySortSpec().getSortKey());
+ }
+ }
+
+ public URI getIndexPath() {
+ return indexPath;
+ }
+
+ public Schema getKeySchema() {
+ return keySchema;
+ }
+
+ public SimplePredicate[] getPredicates() {
+ return predicates;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java
index 8a24add..c8a81ec 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java
@@ -23,6 +23,7 @@ import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.plan.serder.LogicalNodeSerializer;
import org.apache.tajo.plan.serder.PlanProto;
@@ -40,15 +41,16 @@ public class LogicalPlanEqualityTester implements LogicalPlanRewriteRule {
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
return true;
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
+ LogicalPlan plan = context.getPlan();
LogicalNode root = plan.getRootBlock().getRoot();
PlanProto.LogicalNodeTree serialized = LogicalNodeSerializer.serialize(plan.getRootBlock().getRoot());
- LogicalNode deserialized = LogicalNodeDeserializer.deserialize(queryContext, serialized);
+ LogicalNode deserialized = LogicalNodeDeserializer.deserialize(context.getQueryContext(), serialized);
assert root.deepEquals(deserialized);
return plan;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index 7604c53..962ec1f 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -32,6 +32,7 @@ import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.expr.*;
@@ -58,8 +59,8 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) {
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
+ for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) {
for (RelationNode relation : block.getRelations()) {
if (relation.getType() == NodeType.SCAN) {
TableDesc table = ((ScanNode)relation).getTableDesc();
@@ -73,9 +74,10 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
+ LogicalPlan plan = context.getPlan();
LogicalPlan.QueryBlock rootBlock = plan.getRootBlock();
- rewriter.visit(queryContext, plan, rootBlock, rootBlock.getRoot(), new Stack<LogicalNode>());
+ rewriter.visit(context.getQueryContext(), plan, rootBlock, rootBlock.getRoot(), new Stack<LogicalNode>());
return plan;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java
index 64aecf7..ed46f62 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java
@@ -31,6 +31,7 @@ import org.apache.tajo.plan.LogicalPlan.QueryBlock;
import org.apache.tajo.plan.expr.*;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
@@ -56,13 +57,13 @@ public class ProjectionPushDownRule extends
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- LogicalNode toBeOptimized = plan.getRootBlock().getRoot();
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
+ LogicalNode toBeOptimized = context.getPlan().getRootBlock().getRoot();
if (PlannerUtil.checkIfDDLPlan(toBeOptimized)) {
return false;
}
- for (QueryBlock eachBlock: plan.getQueryBlocks()) {
+ for (QueryBlock eachBlock: context.getPlan().getQueryBlocks()) {
if (eachBlock.hasTableExpression()) {
return true;
}
@@ -71,7 +72,8 @@ public class ProjectionPushDownRule extends
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws PlanningException {
+ LogicalPlan plan = rewriteRuleContext.getPlan();
LogicalPlan.QueryBlock rootBlock = plan.getRootBlock();
LogicalPlan.QueryBlock topmostBlock = rootBlock;
@@ -1108,6 +1110,12 @@ public class ProjectionPushDownRule extends
}
@Override
+ public LogicalNode visitIndexScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException {
+ return visitScan(context, plan, block,node, stack);
+ }
+
+ @Override
public LogicalNode visitTableSubQuery(Context upperContext, LogicalPlan plan, LogicalPlan.QueryBlock block,
TableSubQueryNode node, Stack<LogicalNode> stack) throws PlanningException {
Context childContext = new Context(plan, upperContext.requiredSet);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java
new file mode 100644
index 0000000..0c054bd
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite.rules;
+
+import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.statistics.TableStats;
+
+public class SeqScanInfo extends AccessPathInfo {
+ private TableDesc tableDesc;
+
+ public SeqScanInfo(TableStats tableStats) {
+ super(ScanTypeControl.SEQ_SCAN, tableStats);
+ }
+
+ public SeqScanInfo(TableDesc tableDesc) {
+ this(tableDesc.getStats());
+ this.setTableDesc(tableDesc);
+ }
+
+ public TableDesc getTableDesc() {
+ return tableDesc;
+ }
+
+ public void setTableDesc(TableDesc tableDesc) {
+ this.tableDesc = tableDesc;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
index 5cbed7e..9cdf18b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
@@ -36,9 +36,12 @@ import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.expr.FieldEval;
import org.apache.tajo.plan.expr.WindowFunctionEval;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.*;
/**
@@ -129,6 +132,9 @@ public class LogicalNodeDeserializer {
case SCAN:
current = convertScan(context, protoNode);
break;
+ case INDEX_SCAN:
+ current = convertIndexScan(context, protoNode);
+ break;
case CREATE_TABLE:
current = convertCreateTable(nodeMap, protoNode);
@@ -157,6 +163,13 @@ public class LogicalNodeDeserializer {
current = convertTruncateTable(protoNode);
break;
+ case CREATE_INDEX:
+ current = convertCreateIndex(nodeMap, protoNode);
+ break;
+ case DROP_INDEX:
+ current = convertDropIndex(protoNode);
+ break;
+
default:
throw new RuntimeException("Unknown NodeType: " + protoNode.getType().name());
}
@@ -420,7 +433,24 @@ public class LogicalNodeDeserializer {
scan.setOutSchema(convertSchema(protoNode.getOutSchema()));
}
- private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) {
+ private static IndexScanNode convertIndexScan(OverridableConf context, PlanProto.LogicalNode protoNode) {
+ IndexScanNode indexScan = new IndexScanNode(protoNode.getNodeId());
+ fillScanNode(context, protoNode, indexScan);
+
+ PlanProto.IndexScanSpec indexScanSpec = protoNode.getIndexScan();
+ SimplePredicate[] predicates = new SimplePredicate[indexScanSpec.getPredicatesCount()];
+ for (int i = 0; i < predicates.length; i++) {
+ predicates[i] = new SimplePredicate(indexScanSpec.getPredicates(i));
+ }
+
+ indexScan.set(new Schema(indexScanSpec.getKeySchema()), predicates,
+ TUtil.stringToURI(indexScanSpec.getIndexPath()));
+
+ return indexScan;
+ }
+
+ private static PartitionedTableScanNode convertPartitionScan(OverridableConf context,
+ PlanProto.LogicalNode protoNode) {
PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getNodeId());
fillScanNode(context, protoNode, partitionedScan);
@@ -596,6 +626,45 @@ public class LogicalNodeDeserializer {
return truncateTable;
}
+ private static CreateIndexNode convertCreateIndex(Map<Integer, LogicalNode> nodeMap,
+ PlanProto.LogicalNode protoNode) {
+ CreateIndexNode createIndex = new CreateIndexNode(protoNode.getNodeId());
+
+ PlanProto.CreateIndexNode createIndexProto = protoNode.getCreateIndex();
+ createIndex.setIndexName(createIndexProto.getIndexName());
+ createIndex.setIndexMethod(createIndexProto.getIndexMethod());
+ try {
+ createIndex.setIndexPath(new URI(createIndexProto.getIndexPath()));
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ SortSpec[] keySortSpecs = new SortSpec[createIndexProto.getKeySortSpecsCount()];
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ keySortSpecs[i] = new SortSpec(createIndexProto.getKeySortSpecs(i));
+ }
+ createIndex.setKeySortSpecs(new Schema(createIndexProto.getTargetRelationSchema()),
+ keySortSpecs);
+ createIndex.setUnique(createIndexProto.getIsUnique());
+ createIndex.setClustered(createIndexProto.getIsClustered());
+ if (createIndexProto.hasIndexProperties()) {
+ createIndex.setOptions(new KeyValueSet(createIndexProto.getIndexProperties()));
+ }
+ createIndex.setChild(nodeMap.get(createIndexProto.getChildSeq()));
+ createIndex.setInSchema(convertSchema(protoNode.getInSchema()));
+ createIndex.setOutSchema(convertSchema(protoNode.getOutSchema()));
+
+ return createIndex;
+ }
+
+ private static DropIndexNode convertDropIndex(PlanProto.LogicalNode protoNode) {
+ DropIndexNode dropIndex = new DropIndexNode(protoNode.getNodeId());
+
+ PlanProto.DropIndexNode dropIndexProto = protoNode.getDropIndex();
+ dropIndex.setIndexName(dropIndexProto.getIndexName());
+
+ return dropIndex;
+ }
+
private static AggregationFunctionCallEval [] convertAggFuncCallEvals(OverridableConf context,
List<PlanProto.EvalNodeTree> evalTrees) {
AggregationFunctionCallEval [] aggFuncs = new AggregationFunctionCallEval[evalTrees.size()];
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
index 39a13ba..5ad79c0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
@@ -21,12 +21,15 @@ package org.apache.tajo.plan.serder;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.algebra.JoinType;
+import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
import org.apache.tajo.exception.UnimplementedException;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.AddColumn;
import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.RenameColumn;
import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.RenameTable;
@@ -104,6 +107,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
private LogicalNodeTree.Builder treeBuilder = LogicalNodeTree.newBuilder();
}
+ @Override
public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
LogicalRootNode root, Stack<LogicalNode> stack) throws PlanningException {
super.visitRoot(context, plan, block, root, stack);
@@ -138,6 +142,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return node;
}
+ @Override
public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
EvalExprNode exprEval, Stack<LogicalNode> stack) throws PlanningException {
PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder();
@@ -151,6 +156,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return exprEval;
}
+ @Override
public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
ProjectionNode projection, Stack<LogicalNode> stack) throws PlanningException {
super.visitProjection(context, plan, block, projection, stack);
@@ -188,6 +194,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return limit;
}
+ @Override
public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
WindowAggNode windowAgg, Stack<LogicalNode> stack) throws PlanningException {
super.visitWindowAgg(context, plan, block, windowAgg, stack);
@@ -262,6 +269,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return having;
}
+ @Override
public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
GroupbyNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitGroupBy(context, plan, block, node, new Stack<LogicalNode>());
@@ -297,6 +305,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return nodeBuilder;
}
+ @Override
public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
DistinctGroupbyNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitDistinctGroupby(context, plan, block, node, new Stack<LogicalNode>());
@@ -353,6 +362,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return filter;
}
+ @Override
public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode join,
Stack<LogicalNode> stack) throws PlanningException {
super.visitJoin(context, plan, block, join, stack);
@@ -435,6 +445,26 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
}
@Override
+ public LogicalNode visitIndexScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException {
+
+ PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node);
+
+ PlanProto.IndexScanSpec.Builder indexScanSpecBuilder = PlanProto.IndexScanSpec.newBuilder();
+ indexScanSpecBuilder.setKeySchema(node.getKeySchema().getProto());
+ indexScanSpecBuilder.setIndexPath(node.getIndexPath().toString());
+ for (SimplePredicate predicate : node.getPredicates()) {
+ indexScanSpecBuilder.addPredicates(predicate.getProto());
+ }
+
+ PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node);
+ nodeBuilder.setScan(scanBuilder);
+ nodeBuilder.setIndexScan(indexScanSpecBuilder);
+ context.treeBuilder.addNodes(nodeBuilder);
+ return node;
+ }
+
+ @Override
public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
PartitionedTableScanNode node, Stack<LogicalNode> stack)
throws PlanningException {
@@ -458,6 +488,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return node;
}
+ @Override
public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
TableSubQueryNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitTableSubQuery(context, plan, block, node, stack);
@@ -480,6 +511,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return node;
}
+ @Override
public LogicalNode visitCreateTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
CreateTableNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitCreateTable(context, plan, block, node, stack);
@@ -589,6 +621,7 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return node;
}
+ @Override
public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
InsertNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitInsert(context, plan, block, node, stack);
@@ -670,6 +703,47 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
return node;
}
+ @Override
+ public LogicalNode visitCreateIndex(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ CreateIndexNode node, Stack<LogicalNode> stack) throws PlanningException {
+ super.visitCreateIndex(context, plan, block, node, new Stack<LogicalNode>());
+
+ PlanProto.CreateIndexNode.Builder createIndexBuilder = PlanProto.CreateIndexNode.newBuilder();
+ int [] childIds = registerGetChildIds(context, node);
+ createIndexBuilder.setChildSeq(childIds[0]);
+ createIndexBuilder.setIndexName(node.getIndexName());
+ createIndexBuilder.setIndexMethod(node.getIndexMethod());
+ createIndexBuilder.setIndexPath(node.getIndexPath().toString());
+ for (SortSpec sortSpec : node.getKeySortSpecs()) {
+ createIndexBuilder.addKeySortSpecs(sortSpec.getProto());
+ }
+ createIndexBuilder.setTargetRelationSchema(node.getTargetRelationSchema().getProto());
+ createIndexBuilder.setIsUnique(node.isUnique());
+ createIndexBuilder.setIsClustered(node.isClustered());
+ if (node.hasOptions()) {
+ createIndexBuilder.setIndexProperties(node.getOptions().getProto());
+ }
+
+ PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node);
+ nodeBuilder.setCreateIndex(createIndexBuilder);
+ context.treeBuilder.addNodes(nodeBuilder);
+
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitDropIndex(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ DropIndexNode node, Stack<LogicalNode> stack) {
+ PlanProto.DropIndexNode.Builder dropIndexBuilder = PlanProto.DropIndexNode.newBuilder();
+ dropIndexBuilder.setIndexName(node.getIndexName());
+
+ PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node);
+ nodeBuilder.setDropIndex(dropIndexBuilder);
+ context.treeBuilder.addNodes(nodeBuilder);
+
+ return node;
+ }
+
public static PlanProto.NodeType convertType(NodeType type) {
return PlanProto.NodeType.valueOf(type.name());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java
new file mode 100644
index 0000000..73b33d5
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.util;
+
+import org.apache.tajo.catalog.SortSpec;
+import org.apache.tajo.plan.expr.*;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Stack;
+
+public class IndexUtil {
+
+ public static String getIndexName(String indexName , SortSpec[] keys) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(indexName + "_");
+ for(int i = 0 ; i < keys.length ; i ++) {
+ builder.append(keys[i].getSortKey().getSimpleName() + "_");
+ }
+ return builder.toString();
+ }
+
+ public static List<EvalNode> getAllEqualEvals(EvalNode qual) {
+ EvalTreeUtil.EvalFinder finder = new EvalTreeUtil.EvalFinder(EvalType.EQUAL);
+ finder.visitChild(null, qual, new Stack<EvalNode>());
+ return finder.getEvalNodes();
+ }
+
+ private static class FieldAndValueFinder implements EvalNodeVisitor {
+ private LinkedList<BinaryEval> nodeList = new LinkedList<BinaryEval>();
+
+ public LinkedList<BinaryEval> getNodeList () {
+ return this.nodeList;
+ }
+
+ @Override
+ public void visit(EvalNode node) {
+ BinaryEval binaryEval = (BinaryEval) node;
+ switch(node.getType()) {
+ case AND:
+ break;
+ case EQUAL:
+ if( binaryEval.getLeftExpr().getType() == EvalType.FIELD
+ && binaryEval.getRightExpr().getType() == EvalType.CONST ) {
+ nodeList.add(binaryEval);
+ }
+ break;
+ case IS_NULL:
+ if( binaryEval.getLeftExpr().getType() == EvalType.FIELD
+ && binaryEval.getRightExpr().getType() == EvalType.CONST) {
+ nodeList.add(binaryEval);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 0fbd359..ebd47de 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -71,10 +71,31 @@ public class PlannerUtil {
type == NodeType.CREATE_DATABASE ||
type == NodeType.DROP_DATABASE ||
(type == NodeType.CREATE_TABLE && !((CreateTableNode) baseNode).hasSubQuery()) ||
- baseNode.getType() == NodeType.DROP_TABLE ||
- baseNode.getType() == NodeType.ALTER_TABLESPACE ||
- baseNode.getType() == NodeType.ALTER_TABLE ||
- baseNode.getType() == NodeType.TRUNCATE_TABLE;
+ type == NodeType.DROP_TABLE ||
+ type == NodeType.ALTER_TABLESPACE ||
+ type == NodeType.ALTER_TABLE ||
+ type == NodeType.TRUNCATE_TABLE ||
+ type == NodeType.CREATE_INDEX ||
+ type == NodeType.DROP_INDEX;
+ }
+
+ /**
+ * Most update queries require only the updates to the catalog information,
+ * but some queries such as "CREATE INDEX" or CTAS requires distributed execution on multiple cluster nodes.
+ * This function checks whether the given DDL plan requires distributed execution or not.
+ * @param node the root node of a query plan
+ * @return Return true if the input query plan requires distributed execution. Otherwise, return false.
+ */
+ public static boolean isDistExecDDL(LogicalNode node) {
+ LogicalNode baseNode = node;
+ if (node instanceof LogicalRootNode) {
+ baseNode = ((LogicalRootNode) node).getChild();
+ }
+
+ NodeType type = baseNode.getType();
+
+ return type == NodeType.CREATE_INDEX ||
+ type == NodeType.CREATE_TABLE && ((CreateTableNode)baseNode).hasSubQuery();
}
/**
@@ -361,6 +382,12 @@ public class PlannerUtil {
throws PlanningException {
return node;
}
+
+ @Override
+ public LogicalNode visitIndexScan(ReplacerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException {
+ return node;
+ }
}
public static void replaceNode(LogicalNode plan, LogicalNode newNode, NodeType type) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java
index 1ff70a2..9bd3b24 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java
@@ -110,6 +110,9 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
case PARTITIONS_SCAN:
current = visitPartitionedTableScan(context, plan, block, (PartitionedTableScanNode) node, stack);
break;
+ case INDEX_SCAN:
+ current = visitIndexScan(context, plan, block, (IndexScanNode) node, stack);
+ break;
case STORE:
current = visitStoreTable(context, plan, block, (StoreTableNode) node, stack);
break;
@@ -140,6 +143,9 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
case TRUNCATE_TABLE:
current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack);
break;
+ case DROP_INDEX:
+ current = visitDropIndex(context, plan, block, (DropIndexNode) node, stack);
+ break;
default:
throw new PlanningException("Unknown logical node type: " + node.getType());
}
@@ -311,6 +317,12 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
}
@Override
+ public RESULT visitIndexScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, IndexScanNode node,
+ Stack<LogicalNode> stack) throws PlanningException {
+ return null;
+ }
+
+ @Override
public RESULT visitPartitionedTableScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block,
PartitionedTableScanNode node, Stack<LogicalNode> stack)
throws PlanningException {
@@ -387,6 +399,12 @@ public class BasicLogicalPlanVisitor<CONTEXT, RESULT> implements LogicalPlanVisi
}
@Override
+ public RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DropIndexNode node,
+ Stack<LogicalNode> stack) {
+ return null;
+ }
+
+ @Override
public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block,
TruncateTableNode node, Stack<LogicalNode> stack) throws PlanningException {
return null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java
index 73bc7f1..2b33937 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java
@@ -240,6 +240,13 @@ public class ExplainLogicalPlanVisitor extends BasicLogicalPlanVisitor<ExplainLo
return visitUnaryNode(context, plan, block, node, stack);
}
+ @Override
+ public LogicalNode visitDropIndex(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ DropIndexNode node, Stack<LogicalNode> stack) {
+ context.add(context.depth, node.getPlanString());
+ return node;
+ }
+
public static String printDepthString(int maxDepth, DepthString planStr) {
StringBuilder output = new StringBuilder();
String pad = new String(new char[planStr.getDepth() * 3]).replace('\0', ' ');
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java
index d0643c9..a9fb20b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java
@@ -75,6 +75,9 @@ public interface LogicalPlanVisitor<CONTEXT, RESULT> {
RESULT visitScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node,
Stack<LogicalNode> stack) throws PlanningException;
+ RESULT visitIndexScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, IndexScanNode node,
+ Stack<LogicalNode> stack) throws PlanningException;
+
RESULT visitPartitionedTableScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block,
PartitionedTableScanNode node, Stack<LogicalNode> stack) throws PlanningException;
@@ -105,6 +108,9 @@ public interface LogicalPlanVisitor<CONTEXT, RESULT> {
RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, CreateIndexNode node,
Stack<LogicalNode> stack) throws PlanningException;
+ RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DropIndexNode node,
+ Stack<LogicalNode> stack) throws PlanningException;
+
RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node,
Stack<LogicalNode> stack) throws PlanningException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index 3e4f07c..dfee969 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -45,7 +45,7 @@ enum NodeType {
TABLE_SUBQUERY = 15;
SCAN = 16;
PARTITIONS_SCAN = 17;
- BST_INDEX_SCAN = 18;
+ INDEX_SCAN = 18;
STORE = 19;
INSERT = 20;
@@ -56,6 +56,8 @@ enum NodeType {
ALTER_TABLESPACE = 25;
ALTER_TABLE = 26;
TRUNCATE_TABLE = 27;
+ CREATE_INDEX = 28;
+ DROP_INDEX = 29;
}
message LogicalNodeTree {
@@ -71,31 +73,35 @@ message LogicalNode {
optional ScanNode scan = 6;
optional PartitionScanSpec partitionScan = 7;
- optional JoinNode join = 8;
- optional FilterNode filter = 9;
- optional GroupbyNode groupby = 10;
- optional DistinctGroupbyNode distinctGroupby = 11;
- optional SortNode sort = 12;
- optional LimitNode limit = 13;
- optional WindowAggNode windowAgg = 14;
- optional ProjectionNode projection = 15;
- optional EvalExprNode exprEval = 16;
- optional UnionNode union = 17;
- optional TableSubQueryNode tableSubQuery = 18;
- optional PersistentStoreNode persistentStore = 19;
- optional StoreTableNodeSpec storeTable = 20;
- optional InsertNodeSpec insert = 21;
- optional CreateTableNodeSpec createTable = 22;
- optional RootNode root = 23;
- optional SetSessionNode setSession = 24;
-
- optional CreateDatabaseNode createDatabase = 25;
- optional DropDatabaseNode dropDatabase = 26;
- optional DropTableNode dropTable = 27;
-
- optional AlterTablespaceNode alterTablespace = 28;
- optional AlterTableNode alterTable = 29;
- optional TruncateTableNode truncateTableNode = 30;
+ optional IndexScanSpec indexScan = 8;
+ optional JoinNode join = 9;
+ optional FilterNode filter = 10;
+ optional GroupbyNode groupby = 11;
+ optional DistinctGroupbyNode distinctGroupby = 12;
+ optional SortNode sort = 13;
+ optional LimitNode limit = 14;
+ optional WindowAggNode windowAgg = 15;
+ optional ProjectionNode projection = 16;
+ optional EvalExprNode exprEval = 17;
+ optional UnionNode union = 18;
+ optional TableSubQueryNode tableSubQuery = 19;
+ optional PersistentStoreNode persistentStore = 20;
+ optional StoreTableNodeSpec storeTable = 21;
+ optional InsertNodeSpec insert = 22;
+ optional CreateTableNodeSpec createTable = 23;
+ optional RootNode root = 24;
+ optional SetSessionNode setSession = 25;
+
+ optional CreateDatabaseNode createDatabase = 26;
+ optional DropDatabaseNode dropDatabase = 27;
+ optional DropTableNode dropTable = 28;
+
+ optional AlterTablespaceNode alterTablespace = 29;
+ optional AlterTableNode alterTable = 30;
+ optional TruncateTableNode truncateTableNode = 31;
+
+ optional CreateIndexNode createIndex = 32;
+ optional DropIndexNode dropIndex = 33;
}
message ScanNode {
@@ -110,6 +116,17 @@ message PartitionScanSpec {
repeated string paths = 1;
}
+message IndexScanSpec {
+ required SchemaProto keySchema = 1;
+ required string indexPath = 2;
+ repeated SimplePredicateProto predicates = 3;
+}
+
+message SimplePredicateProto {
+ required SortSpecProto keySortSpec = 1;
+ required Datum value = 2;
+}
+
message FilterNode {
required int32 childSeq = 1;
required EvalNodeTree qual = 2;
@@ -301,6 +318,22 @@ message AlterTableNode {
optional AddColumn addColumn = 5;
}
+message CreateIndexNode {
+ required int32 childSeq = 1;
+ required string indexName = 2;
+ required IndexMethod indexMethod = 3;
+ required string indexPath = 4;
+ repeated SortSpecProto keySortSpecs = 5;
+ required SchemaProto targetRelationSchema = 6;
+ optional bool isUnique = 7 [default = false];
+ optional bool isClustered = 8 [default = false];
+ optional KeyValueSetProto indexProperties = 9;
+}
+
+message DropIndexNode {
+ required string indexName = 1;
+}
+
enum EvalType {
NOT = 0;
AND = 1;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
index e95aeec..f9a57fa 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java
@@ -32,6 +32,7 @@ import org.apache.tajo.plan.logical.SortNode;
import org.apache.tajo.plan.logical.SortNode.SortPurpose;
import org.apache.tajo.plan.logical.UnaryNode;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.util.PlannerUtil;
public class AddSortForInsertRewriter implements LogicalPlanRewriteRule {
@@ -54,13 +55,14 @@ public class AddSortForInsertRewriter implements LogicalPlanRewriteRule {
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
- StoreType storeType = PlannerUtil.getStoreType(plan);
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
+ StoreType storeType = PlannerUtil.getStoreType(context.getPlan());
return storeType != null;
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
+ LogicalPlan plan = context.getPlan();
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
UnaryNode insertNode = rootNode.getChild();
LogicalNode childNode = insertNode.getChild();
[3/5] tajo git commit: TAJO-838: Improve query planner to utilize
index. (jihoon)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java
index c6466f5..9187a69 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java
@@ -33,15 +33,7 @@ import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.conf.TajoConf;
@@ -297,43 +289,35 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
return tuples;
}
-
+
private List<Tuple> getIndexes(Schema outSchema) {
- List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
+ List<IndexDescProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
List<Column> columns = outSchema.getColumns();
Tuple aTuple;
-
- for (IndexProto index: indexList) {
+
+ for (IndexDescProto index: indexList) {
aTuple = new VTuple(outSchema.size());
-
+
for (int fieldId = 0; fieldId < columns.size(); fieldId++) {
Column column = columns.get(fieldId);
-
+
if ("db_id".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId()));
+ aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getDbId()));
} else if ("tid".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createInt4(index.getTId()));
+ aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getTid()));
} else if ("index_name".equalsIgnoreCase(column.getSimpleName())) {
aTuple.put(fieldId, DatumFactory.createText(index.getIndexName()));
- } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getColumnName()));
- } else if ("data_type".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getDataType()));
- } else if ("index_type".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createText(index.getIndexType()));
- } else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique()));
- } else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered()));
- } else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) {
- aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending()));
+ } else if ("index_method".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createText(index.getIndexMethod().name()));
+ } else if ("index_path".equalsIgnoreCase(column.getSimpleName())) {
+ aTuple.put(fieldId, DatumFactory.createText(index.getIndexPath()));
}
}
-
+
tuples.add(aTuple);
}
-
+
return tuples;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 249d335..f05f54b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -36,6 +36,7 @@ import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.NoSuchDatabaseException;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.query.QueryContext;
@@ -57,6 +58,7 @@ import org.apache.tajo.rpc.BlockingRpcServer;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto;
+import org.apache.tajo.util.IPCUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.util.ProtoUtil;
@@ -143,19 +145,17 @@ public class TajoMasterClientService extends AbstractService {
String sessionId =
context.getSessionManager().createSession(request.getUsername(), databaseName);
CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder();
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
builder.setSessionId(TajoIdProtos.SessionIdProto.newBuilder().setId(sessionId).build());
builder.setSessionVars(ProtoUtil.convertFromMap(context.getSessionManager().getAllVariables(sessionId)));
return builder.build();
} catch (NoSuchDatabaseException nsde) {
CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder();
- builder.setResultCode(ResultCode.ERROR);
- builder.setMessage(nsde.getMessage());
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, nsde.getMessage(), null));
return builder.build();
} catch (InvalidSessionException e) {
CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder();
- builder.setResultCode(ResultCode.ERROR);
- builder.setMessage(e.getMessage());
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, e.getMessage(), null));
return builder.build();
}
}
@@ -173,15 +173,14 @@ public class TajoMasterClientService extends AbstractService {
public SessionUpdateResponse buildSessionUpdateOnSuccess(Map<String, String> variables) {
SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder();
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
builder.setSessionVars(new KeyValueSet(variables).getProto());
return builder.build();
}
public SessionUpdateResponse buildSessionUpdateOnError(String message) {
SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder();
- builder.setResultCode(ResultCode.ERROR);
- builder.setMessage(message);
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, message, null));
return builder.build();
}
@@ -288,12 +287,8 @@ public class TajoMasterClientService extends AbstractService {
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setIsForwarded(true);
responseBuilder.setUserName(context.getConf().getVar(ConfVars.USERNAME));
- responseBuilder.setResultCode(ResultCode.ERROR);
- if (e.getMessage() != null) {
- responseBuilder.setErrorMessage(ExceptionUtils.getStackTrace(e));
- } else {
- responseBuilder.setErrorMessage("Internal Error");
- }
+ responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ e.getMessage() == null ? "Internal Error" : ExceptionUtils.getStackTrace(e), null));
return responseBuilder.build();
}
}
@@ -304,23 +299,15 @@ public class TajoMasterClientService extends AbstractService {
try {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
QueryContext queryContext = new QueryContext(conf, session);
- if (queryContext.getCurrentDatabase() == null) {
- for (Map.Entry<String,String> e : queryContext.getAllKeyValus().entrySet()) {
- System.out.println(e.getKey() + "=" + e.getValue());
- }
- }
- UpdateQueryResponse.Builder builder = UpdateQueryResponse.newBuilder();
+ UpdateQueryResponse.Builder responseBuilder = UpdateQueryResponse.newBuilder();
try {
context.getGlobalEngine().updateQuery(queryContext, request.getQuery(), request.getIsJson());
- builder.setResultCode(ResultCode.OK);
- return builder.build();
+ return responseBuilder.setResult(IPCUtil.buildOkRequestResult()).build();
} catch (Exception e) {
- builder.setResultCode(ResultCode.ERROR);
- if (e.getMessage() == null) {
- builder.setErrorMessage(ExceptionUtils.getStackTrace(e));
- }
- return builder.build();
+ responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ e.getMessage() == null ? ExceptionUtils.getStackTrace(e) : null, null));
+ return responseBuilder.build();
}
} catch (Throwable t) {
throw new ServiceException(t);
@@ -460,7 +447,7 @@ public class TajoMasterClientService extends AbstractService {
builder.setQueryId(request.getQueryId());
if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) {
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED);
} else {
QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId);
@@ -474,7 +461,7 @@ public class TajoMasterClientService extends AbstractService {
}
if (queryInfo != null) {
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
builder.setState(queryInfo.getQueryState());
boolean isCreateTable = queryInfo.getQueryContext().isCreateTable();
@@ -495,11 +482,11 @@ public class TajoMasterClientService extends AbstractService {
} else {
Session session = context.getSessionManager().getSession(request.getSessionId().getId());
if (session.getNonForwardQueryResultScanner(queryId) != null) {
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED);
} else {
- builder.setResultCode(ResultCode.ERROR);
- builder.setErrorMessage("No such query: " + queryId.toString());
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ "No such query: " + queryId.toString(), null));
}
}
}
@@ -531,17 +518,16 @@ public class TajoMasterClientService extends AbstractService {
resultSetBuilder.addAllSerializedTuples(rows);
builder.setResultSet(resultSetBuilder.build());
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
LOG.info("Send result to client for " +
request.getSessionId().getId() + "," + queryId + ", " + rows.size() + " rows");
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
- builder.setResultCode(ResultCode.ERROR);
String errorMessage = t.getMessage() == null ? t.getClass().getName() : t.getMessage();
- builder.setErrorMessage(errorMessage);
- builder.setErrorTrace(org.apache.hadoop.util.StringUtils.stringifyException(t));
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ errorMessage, org.apache.hadoop.util.StringUtils.stringifyException(t)));
}
return builder.build();
}
@@ -581,11 +567,11 @@ public class TajoMasterClientService extends AbstractService {
if (queryInfo != null) {
builder.setQueryInfo(queryInfo.getProto());
}
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(IPCUtil.buildOkRequestResult());
} catch (Throwable t) {
LOG.warn(t.getMessage(), t);
- builder.setResultCode(ResultCode.ERROR);
- builder.setErrorMessage(org.apache.hadoop.util.StringUtils.stringifyException(t));
+ builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ org.apache.hadoop.util.StringUtils.stringifyException(t), null));
}
return builder.build();
@@ -779,13 +765,13 @@ public class TajoMasterClientService extends AbstractService {
if (catalog.existsTable(databaseName, tableName)) {
return TableResponse.newBuilder()
- .setResultCode(ResultCode.OK)
+ .setResult(IPCUtil.buildOkRequestResult())
.setTableDesc(catalog.getTableDesc(databaseName, tableName).getProto())
.build();
} else {
return TableResponse.newBuilder()
- .setResultCode(ResultCode.ERROR)
- .setErrorMessage("ERROR: no such a table: " + request.getTableName())
+ .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ "ERROR: no such a table: " + request.getTableName(), null))
.build();
}
} catch (Throwable t) {
@@ -821,21 +807,21 @@ public class TajoMasterClientService extends AbstractService {
meta, path, true, partitionDesc, false);
} catch (Exception e) {
return TableResponse.newBuilder()
- .setResultCode(ResultCode.ERROR)
- .setErrorMessage(e.getMessage()).build();
+ .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, e.getMessage(), null))
+ .build();
}
return TableResponse.newBuilder()
- .setResultCode(ResultCode.OK)
+ .setResult(IPCUtil.buildOkRequestResult())
.setTableDesc(desc.getProto()).build();
} catch (InvalidSessionException ise) {
return TableResponse.newBuilder()
- .setResultCode(ResultCode.ERROR)
- .setErrorMessage(ise.getMessage()).build();
+ .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, ise.getMessage(), null))
+ .build();
} catch (IOException ioe) {
return TableResponse.newBuilder()
- .setResultCode(ResultCode.ERROR)
- .setErrorMessage(ioe.getMessage()).build();
+ .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, ioe.getMessage(), null))
+ .build();
}
}
@@ -875,12 +861,184 @@ public class TajoMasterClientService extends AbstractService {
}
}
return FunctionResponse.newBuilder()
- .setResultCode(ResultCode.OK)
+ .setResult(IPCUtil.buildOkRequestResult())
.addAllFunctions(functionProtos)
.build();
} catch (Throwable t) {
throw new ServiceException(t);
}
}
+
+ @Override
+ public IndexDescProto getIndexWithName(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+ return catalog.getIndexByName(databaseName, indexName).getProto();
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public BoolProto existIndexWithName(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+ return catalog.existIndexByName(databaseName, indexName) ?
+ ProtoUtil.TRUE : ProtoUtil.FALSE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public GetIndexesResponse getIndexesForTable(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getValue();
+ }
+
+ GetIndexesResponse.Builder builder = GetIndexesResponse.newBuilder();
+ for (IndexDesc index : catalog.getAllIndexesByTable(databaseName, tableName)) {
+ builder.addIndexes(index.getProto());
+ }
+ builder.setResult(IPCUtil.buildOkRequestResult());
+ return builder.build();
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public BoolProto existIndexesForTable(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getValue();
+ }
+ return catalog.existIndexesByTable(databaseName, tableName) ?
+ ProtoUtil.TRUE : ProtoUtil.FALSE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public GetIndexWithColumnsResponse getIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getTableName())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getTableName());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getTableName();
+ }
+ String[] columnNames = new String[request.getColumnNamesCount()];
+ columnNames = request.getColumnNamesList().toArray(columnNames);
+
+ GetIndexWithColumnsResponse.Builder builder = GetIndexWithColumnsResponse.newBuilder();
+ builder.setResult(IPCUtil.buildOkRequestResult());
+ builder.setIndexDesc(catalog.getIndexByColumnNames(databaseName, tableName, columnNames).getProto());
+ return builder.build();
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public BoolProto existIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String tableName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getTableName())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getTableName());
+ databaseName = splitted[0];
+ tableName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ tableName = request.getTableName();
+ }
+ String[] columnNames = new String[request.getColumnNamesCount()];
+ columnNames = request.getColumnNamesList().toArray(columnNames);
+ return catalog.existIndexByColumnNames(databaseName, tableName, columnNames) ?
+ ProtoUtil.TRUE : ProtoUtil.FALSE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
+
+ @Override
+ public BoolProto dropIndex(RpcController controller, SessionedStringProto request)
+ throws ServiceException {
+ try {
+ context.getSessionManager().touch(request.getSessionId().getId());
+ Session session = context.getSessionManager().getSession(request.getSessionId().getId());
+
+ String indexName, databaseName;
+ if (CatalogUtil.isFQTableName(request.getValue())) {
+ String [] splitted = CatalogUtil.splitFQTableName(request.getValue());
+ databaseName = splitted[0];
+ indexName = splitted[1];
+ } else {
+ databaseName = session.getCurrentDatabase();
+ indexName = request.getValue();
+ }
+ return catalog.dropIndex(databaseName, indexName) ?
+ ProtoUtil.TRUE : ProtoUtil.FALSE;
+ } catch (Throwable t) {
+ throw new ServiceException(t);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index acbaa01..5b12438 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -68,45 +68,88 @@ public class DDLExecutor {
switch (root.getType()) {
- case ALTER_TABLESPACE:
- AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root;
- alterTablespace(context, queryContext, alterTablespace);
- return true;
-
-
- case CREATE_DATABASE:
- CreateDatabaseNode createDatabase = (CreateDatabaseNode) root;
- createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists());
- return true;
- case DROP_DATABASE:
- DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root;
- dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists());
- return true;
-
-
- case CREATE_TABLE:
- CreateTableNode createTable = (CreateTableNode) root;
- createTable(queryContext, createTable, createTable.isIfNotExists());
- return true;
- case DROP_TABLE:
- DropTableNode dropTable = (DropTableNode) root;
- dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge());
- return true;
- case TRUNCATE_TABLE:
- TruncateTableNode truncateTable = (TruncateTableNode) root;
- truncateTable(queryContext, truncateTable);
- return true;
-
- case ALTER_TABLE:
- AlterTableNode alterTable = (AlterTableNode) root;
- alterTable(context, queryContext, alterTable);
- return true;
+ case ALTER_TABLESPACE:
+ AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root;
+ alterTablespace(context, queryContext, alterTablespace);
+ return true;
+
+
+ case CREATE_DATABASE:
+ CreateDatabaseNode createDatabase = (CreateDatabaseNode) root;
+ createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists());
+ return true;
+ case DROP_DATABASE:
+ DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root;
+ dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists());
+ return true;
+
+
+ case CREATE_TABLE:
+ CreateTableNode createTable = (CreateTableNode) root;
+ createTable(queryContext, createTable, createTable.isIfNotExists());
+ return true;
+ case DROP_TABLE:
+ DropTableNode dropTable = (DropTableNode) root;
+ dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge());
+ return true;
+ case TRUNCATE_TABLE:
+ TruncateTableNode truncateTable = (TruncateTableNode) root;
+ truncateTable(queryContext, truncateTable);
+ return true;
+
+ case ALTER_TABLE:
+ AlterTableNode alterTable = (AlterTableNode) root;
+ alterTable(context, queryContext, alterTable);
+ return true;
+
+ case CREATE_INDEX:
+ // The catalog information for the created index is automatically updated when the query is successfully finished.
+ // See the Query.CreateIndexHook class.
+ return true;
+
+ case DROP_INDEX:
+ DropIndexNode dropIndexNode = (DropIndexNode) root;
+ dropIndex(queryContext, dropIndexNode);
+ return true;
default:
throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson());
}
}
+ public void dropIndex(final QueryContext queryContext, final DropIndexNode dropIndexNode) {
+ String databaseName, simpleIndexName;
+ if (CatalogUtil.isFQTableName(dropIndexNode.getIndexName())) {
+ String [] splits = CatalogUtil.splitFQTableName(dropIndexNode.getIndexName());
+ databaseName = splits[0];
+ simpleIndexName = splits[1];
+ } else {
+ databaseName = queryContext.getCurrentDatabase();
+ simpleIndexName = dropIndexNode.getIndexName();
+ }
+
+ if (!catalog.existIndexByName(databaseName, simpleIndexName)) {
+ throw new NoSuchIndexException(simpleIndexName);
+ }
+
+ IndexDesc desc = catalog.getIndexByName(databaseName, simpleIndexName);
+
+ if (!catalog.dropIndex(databaseName, simpleIndexName)) {
+ LOG.info("Cannot drop index \"" + simpleIndexName + "\".");
+ throw new CatalogException("Cannot drop index \"" + simpleIndexName + "\".");
+ }
+
+ Path indexPath = new Path(desc.getIndexPath());
+ try {
+ FileSystem fs = indexPath.getFileSystem(context.getConf());
+ fs.delete(indexPath, true);
+ } catch (IOException e) {
+ throw new InternalError(e.getMessage());
+ }
+
+ LOG.info("Index " + simpleIndexName + " is dropped.");
+ }
+
/**
* Alter a given table
*/
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
index 2242445..26476a3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java
@@ -29,8 +29,10 @@ import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.catalog.exception.AlreadyExistsIndexException;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
@@ -40,6 +42,7 @@ import org.apache.tajo.engine.planner.physical.EvalExprExec;
import org.apache.tajo.engine.planner.physical.StoreTableExec;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.ClientProtos;
+import org.apache.tajo.ipc.ClientProtos.ResultCode;
import org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse;
import org.apache.tajo.master.NonForwardQueryResultFileScanner;
import org.apache.tajo.master.NonForwardQueryResultScanner;
@@ -58,6 +61,7 @@ import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.storage.*;
+import org.apache.tajo.util.IPCUtil;
import org.apache.tajo.util.ProtoUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -98,10 +102,18 @@ public class QueryExecutor {
} else if (PlannerUtil.checkIfDDLPlan(rootNode)) {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
- ddlExecutor.execute(queryContext, plan);
- response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ if (PlannerUtil.isDistExecDDL(rootNode)) {
+ if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) {
+ checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild());
+ }
+ executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
+ } else {
+ response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
+ response.setResult(IPCUtil.buildOkRequestResult());
+ }
+
+ ddlExecutor.execute(queryContext, plan);
} else if (plan.isExplain()) { // explain query
execExplain(plan, response);
@@ -142,8 +154,8 @@ public class QueryExecutor {
session.selectDatabase(setSessionNode.getValue());
} else {
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- response.setResultCode(ClientProtos.ResultCode.ERROR);
- response.setErrorMessage("database \"" + databaseName + "\" does not exists.");
+ response.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ "database \"" + databaseName + "\" does not exists.", null));
}
// others
@@ -157,7 +169,7 @@ public class QueryExecutor {
context.getSystemMetrics().counter("Query", "numDDLQuery").inc();
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ response.setResult(IPCUtil.buildOkRequestResult());
}
public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) throws IOException {
@@ -183,7 +195,7 @@ public class QueryExecutor {
response.setResultSet(serializedResBuilder.build());
response.setMaxRowNum(lines.length);
- response.setResultCode(ClientProtos.ResultCode.OK);
+ response.setResult(IPCUtil.buildOkRequestResult());
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
}
@@ -205,7 +217,7 @@ public class QueryExecutor {
response.setQueryId(queryId.getProto());
response.setMaxRowNum(maxRow);
response.setTableDesc(queryResultScanner.getTableDesc().getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ response.setResult(IPCUtil.buildOkRequestResult());
}
public void execSimpleQuery(QueryContext queryContext, Session session, String query, LogicalPlan plan,
@@ -235,7 +247,7 @@ public class QueryExecutor {
response.setQueryId(queryId.getProto());
response.setMaxRowNum(maxRow);
response.setTableDesc(desc.getProto());
- response.setResultCode(ClientProtos.ResultCode.OK);
+ response.setResult(IPCUtil.buildOkRequestResult());
}
public void execNonFromQuery(QueryContext queryContext, Session session, String query,
@@ -267,7 +279,7 @@ public class QueryExecutor {
responseBuilder.setResultSet(serializedResBuilder);
responseBuilder.setMaxRowNum(1);
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ responseBuilder.setResult(IPCUtil.buildOkRequestResult());
}
}
@@ -369,7 +381,7 @@ public class QueryExecutor {
// If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted rows.
responseBuilder.setMaxRowNum(-1);
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ responseBuilder.setResult(IPCUtil.buildOkRequestResult());
}
public void executeDistributedQuery(QueryContext queryContext, Session session,
@@ -398,13 +410,13 @@ public class QueryExecutor {
if(queryInfo == null) {
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
- responseBuilder.setErrorMessage("Fail starting QueryMaster.");
+ responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ "Fail starting QueryMaster.", null));
LOG.error("Fail starting QueryMaster: " + sql);
} else {
responseBuilder.setIsForwarded(true);
responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
- responseBuilder.setResultCode(ClientProtos.ResultCode.OK);
+ responseBuilder.setResult(IPCUtil.buildOkRequestResult());
if(queryInfo.getQueryMasterHost() != null) {
responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
}
@@ -413,4 +425,23 @@ public class QueryExecutor {
" is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort());
}
}
+
+ private void checkIndexExistence(final QueryContext queryContext, final CreateIndexNode createIndexNode)
+ throws IOException {
+ String databaseName, simpleIndexName, qualifiedIndexName;
+ if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) {
+ String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
+ databaseName = splits[0];
+ simpleIndexName = splits[1];
+ qualifiedIndexName = createIndexNode.getIndexName();
+ } else {
+ databaseName = queryContext.getCurrentDatabase();
+ simpleIndexName = createIndexNode.getIndexName();
+ qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName);
+ }
+
+ if (catalog.existIndexByName(databaseName, simpleIndexName)) {
+ throw new AlreadyExistsIndexException(qualifiedIndexName);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
index a626df1..88ecebc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -32,6 +32,8 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryId;
import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoProtos.QueryState;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.TableDesc;
@@ -472,6 +474,7 @@ public class Query implements EventHandler<QueryEvent> {
hookList.add(new MaterializedResultHook());
hookList.add(new CreateTableHook());
hookList.add(new InsertTableHook());
+ hookList.add(new CreateIndexHook());
}
public void execute(QueryContext queryContext, Query query,
@@ -485,6 +488,48 @@ public class Query implements EventHandler<QueryEvent> {
}
}
+ private class CreateIndexHook implements QueryHook {
+
+ @Override
+ public boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) {
+ Stage lastStage = query.getStage(finalExecBlockId);
+ return lastStage.getBlock().getPlan().getType() == NodeType.CREATE_INDEX;
+ }
+
+ @Override
+ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception {
+ CatalogService catalog = context.getWorkerContext().getCatalog();
+ Stage lastStage = query.getStage(finalExecBlockId);
+
+ CreateIndexNode createIndexNode = (CreateIndexNode) lastStage.getBlock().getPlan();
+ String databaseName, simpleIndexName, qualifiedIndexName;
+ if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) {
+ String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName());
+ databaseName = splits[0];
+ simpleIndexName = splits[1];
+ qualifiedIndexName = createIndexNode.getIndexName();
+ } else {
+ databaseName = queryContext.getCurrentDatabase();
+ simpleIndexName = createIndexNode.getIndexName();
+ qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName);
+ }
+ ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN);
+ if (scanNode == null) {
+ throw new IOException("Cannot find the table of the relation");
+ }
+ IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(),
+ simpleIndexName, createIndexNode.getIndexPath(),
+ createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(),
+ createIndexNode.isUnique(), false, scanNode.getLogicalSchema());
+ if (catalog.createIndex(indexDesc)) {
+ LOG.info("Index " + qualifiedIndexName + " is created for the table " + scanNode.getTableName() + ".");
+ } else {
+ LOG.info("Index creation " + qualifiedIndexName + " is failed.");
+ throw new CatalogException("Cannot create index \"" + qualifiedIndexName + "\".");
+ }
+ }
+ }
+
private class MaterializedResultHook implements QueryHook {
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 9c789a5..a125415 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -38,6 +38,12 @@ import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.plan.LogicalOptimizer;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.logical.LogicalRootNode;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.UnimplementedException;
@@ -48,15 +54,9 @@ import org.apache.tajo.master.TajoContainerProxy;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.rm.TajoWorkerResourceManager;
import org.apache.tajo.master.session.Session;
-import org.apache.tajo.plan.LogicalOptimizer;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.plan.logical.LogicalRootNode;
import org.apache.tajo.plan.logical.NodeType;
import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
@@ -347,9 +347,10 @@ public class QueryMasterTask extends CompositeService {
LOG.warn("Query already started");
return;
}
+ LOG.info(SessionVars.INDEX_ENABLED.keyname() + " : " + queryContext.getBool(SessionVars.INDEX_ENABLED));
CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog();
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(systemConf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog);
Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class);
jsonExpr = null; // remove the possible OOM
plan = planner.createPlan(queryContext, expr);
@@ -393,6 +394,14 @@ public class QueryMasterTask extends CompositeService {
tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
}
}
+
+ scanNodes = PlannerUtil.findAllNodes(block.getRoot(), NodeType.INDEX_SCAN);
+ if (scanNodes != null) {
+ for (LogicalNode eachScanNode : scanNodes) {
+ ScanNode scanNode = (ScanNode) eachScanNode;
+ tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+ }
+ }
}
MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan);
queryMasterContext.getGlobalPlanner().build(masterPlan);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java
new file mode 100644
index 0000000..dcffe62
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.util;
+
+import org.apache.tajo.annotation.Nullable;
+import org.apache.tajo.ipc.ClientProtos.RequestResult;
+import org.apache.tajo.ipc.ClientProtos.ResultCode;
+
+public class IPCUtil {
+
+ public static RequestResult buildOkRequestResult() {
+ return buildRequestResult(ResultCode.OK, null, null);
+ }
+
+ public static RequestResult buildRequestResult(ResultCode code,
+ @Nullable String errorMessage,
+ @Nullable String errorTrace) {
+ RequestResult.Builder builder = RequestResult.newBuilder();
+ builder.setResultCode(code);
+ if (errorMessage != null) {
+ builder.setErrorMessage(errorMessage);
+ }
+ if (errorTrace != null) {
+ builder.setErrorTrace(errorTrace);
+ }
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java
deleted file mode 100644
index 3147bb6..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.util;
-
-import com.google.gson.Gson;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.engine.json.CoreGsonHelper;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.expr.*;
-import org.apache.tajo.plan.logical.IndexScanNode;
-import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.storage.fragment.FileFragment;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map.Entry;
-
-public class IndexUtil {
- public static String getIndexNameOfFrag(FileFragment fragment, SortSpec[] keys) {
- StringBuilder builder = new StringBuilder();
- builder.append(fragment.getPath().getName() + "_");
- builder.append(fragment.getStartKey() + "_" + fragment.getLength() + "_");
- for(int i = 0 ; i < keys.length ; i ++) {
- builder.append(keys[i].getSortKey().getSimpleName()+"_");
- }
- builder.append("_index");
- return builder.toString();
-
- }
-
- public static String getIndexName(String indexName , SortSpec[] keys) {
- StringBuilder builder = new StringBuilder();
- builder.append(indexName + "_");
- for(int i = 0 ; i < keys.length ; i ++) {
- builder.append(keys[i].getSortKey().getSimpleName() + "_");
- }
- return builder.toString();
- }
-
- public static IndexScanNode indexEval(LogicalPlan plan, ScanNode scanNode,
- Iterator<Entry<String, String>> iter ) {
-
- EvalNode qual = scanNode.getQual();
- Gson gson = CoreGsonHelper.getInstance();
-
- FieldAndValueFinder nodeFinder = new FieldAndValueFinder();
- qual.preOrder(nodeFinder);
- LinkedList<BinaryEval> nodeList = nodeFinder.getNodeList();
-
- int maxSize = Integer.MIN_VALUE;
- SortSpec[] maxIndex = null;
-
- String json;
- while(iter.hasNext()) {
- Entry<String , String> entry = iter.next();
- json = entry.getValue();
- SortSpec[] sortKey = gson.fromJson(json, SortSpec[].class);
- if(sortKey.length > nodeList.size()) {
- /* If the number of the sort key is greater than where condition,
- * this index cannot be used
- * */
- continue;
- } else {
- boolean[] equal = new boolean[sortKey.length];
- for(int i = 0 ; i < sortKey.length ; i ++) {
- for(int j = 0 ; j < nodeList.size() ; j ++) {
- Column col = ((FieldEval)(nodeList.get(j).getLeftExpr())).getColumnRef();
- if(col.equals(sortKey[i].getSortKey())) {
- equal[i] = true;
- }
- }
- }
- boolean chk = true;
- for(int i = 0 ; i < equal.length ; i ++) {
- chk = chk && equal[i];
- }
- if(chk) {
- if(maxSize < sortKey.length) {
- maxSize = sortKey.length;
- maxIndex = sortKey;
- }
- }
- }
- }
- if(maxIndex == null) {
- return null;
- } else {
- Schema keySchema = new Schema();
- for(int i = 0 ; i < maxIndex.length ; i ++ ) {
- keySchema.addColumn(maxIndex[i].getSortKey());
- }
- Datum[] datum = new Datum[nodeList.size()];
- for(int i = 0 ; i < nodeList.size() ; i ++ ) {
- datum[i] = ((ConstEval)(nodeList.get(i).getRightExpr())).getValue();
- }
-
- return new IndexScanNode(plan.newPID(), scanNode, keySchema , datum , maxIndex);
- }
-
- }
-
-
- private static class FieldAndValueFinder implements EvalNodeVisitor {
- private LinkedList<BinaryEval> nodeList = new LinkedList<BinaryEval>();
-
- public LinkedList<BinaryEval> getNodeList () {
- return this.nodeList;
- }
-
- @Override
- public void visit(EvalNode node) {
- BinaryEval binaryEval = (BinaryEval) node;
- switch(node.getType()) {
- case AND:
- break;
- case EQUAL:
- if( binaryEval.getLeftExpr().getType() == EvalType.FIELD
- && binaryEval.getRightExpr().getType() == EvalType.CONST ) {
- nodeList.add(binaryEval);
- }
- break;
- case IS_NULL:
- if( binaryEval.getLeftExpr().getType() == EvalType.FIELD
- && binaryEval.getRightExpr().getType() == EvalType.CONST) {
- nodeList.add(binaryEval);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
index da25fe6..cd9e6ef 100644
--- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
@@ -304,7 +304,7 @@ public class QueryExecutorServlet extends HttpServlet {
LOG.error("Internal Error: SubmissionResponse is NULL");
error = new Exception("Internal Error: SubmissionResponse is NULL");
- } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
queryId = new QueryId(response.getQueryId());
getQueryResult(queryId);
@@ -316,9 +316,9 @@ public class QueryExecutorServlet extends HttpServlet {
progress.set(100);
}
- } else if (response.getResultCode() == ClientProtos.ResultCode.ERROR) {
- if (response.hasErrorMessage()) {
- StringBuffer errorMessage = new StringBuffer(response.getErrorMessage());
+ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) {
+ if (response.getResult().hasErrorMessage()) {
+ StringBuffer errorMessage = new StringBuffer(response.getResult().getErrorMessage());
String modifiedMessage;
if (errorMessage.length() > 200) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
index 1c83110..6a13898 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java
@@ -25,10 +25,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
import org.apache.tajo.QueryId;
+import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.ipc.ClientProtos.GetQueryHistoryResponse;
import org.apache.tajo.ipc.ClientProtos.QueryIdRequest;
+import org.apache.tajo.ipc.ClientProtos.RequestResult;
import org.apache.tajo.ipc.ClientProtos.ResultCode;
import org.apache.tajo.ipc.QueryMasterClientProtocol;
import org.apache.tajo.master.querymaster.QueryMasterTask;
@@ -129,14 +132,32 @@ public class TajoWorkerClientService extends AbstractService {
if (queryHistory != null) {
builder.setQueryHistory(queryHistory.getProto());
}
- builder.setResultCode(ResultCode.OK);
+ builder.setResult(buildOkRequestResult());
} catch (Throwable t) {
LOG.warn(t.getMessage(), t);
- builder.setResultCode(ResultCode.ERROR);
- builder.setErrorMessage(org.apache.hadoop.util.StringUtils.stringifyException(t));
+ builder.setResult(buildRequestResult(ResultCode.ERROR,
+ StringUtils.stringifyException(t), null));
}
return builder.build();
}
+
+ private RequestResult buildOkRequestResult() {
+ return buildRequestResult(ResultCode.OK, null, null);
+ }
+
+ private RequestResult buildRequestResult(ResultCode code,
+ @Nullable String errorMessage,
+ @Nullable String errorTrace) {
+ RequestResult.Builder builder = RequestResult.newBuilder();
+ builder.setResultCode(code);
+ if (errorMessage != null) {
+ builder.setErrorMessage(errorMessage);
+ }
+ if (errorTrace != null) {
+ builder.setErrorTrace(errorTrace);
+ }
+ return builder.build();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
index 5f9c6ac..b784c64 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
@@ -125,21 +125,9 @@ public class Task {
this.inputStats = new TableStats();
plan = LogicalNodeDeserializer.deserialize(queryContext, request.getPlan());
- LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN);
- if (scanNode != null) {
- for (LogicalNode node : scanNode) {
- ScanNode scan = (ScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
-
- LogicalNode [] partitionScanNode = PlannerUtil.findAllNodes(plan, NodeType.PARTITIONS_SCAN);
- if (partitionScanNode != null) {
- for (LogicalNode node : partitionScanNode) {
- PartitionedTableScanNode scan = (PartitionedTableScanNode) node;
- descs.put(scan.getCanonicalName(), scan.getTableDesc());
- }
- }
+ updateDescsForScanNodes(NodeType.SCAN);
+ updateDescsForScanNodes(NodeType.PARTITIONS_SCAN);
+ updateDescsForScanNodes(NodeType.INDEX_SCAN);
interQuery = request.getProto().getInterQuery();
if (interQuery) {
@@ -181,6 +169,17 @@ public class Task {
LOG.info("==================================");
}
+ private void updateDescsForScanNodes(NodeType nodeType) {
+ assert nodeType == NodeType.SCAN || nodeType == NodeType.PARTITIONS_SCAN || nodeType == NodeType.INDEX_SCAN;
+ LogicalNode[] scanNodes = PlannerUtil.findAllNodes(plan, nodeType);
+ if (scanNodes != null) {
+ for (LogicalNode node : scanNodes) {
+ ScanNode scanNode = (ScanNode) node;
+ descs.put(scanNode.getCanonicalName(), scanNode.getTableDesc());
+ }
+ }
+ }
+
public void init() throws IOException {
if (context.getState() == TaskAttemptState.TA_PENDING) {
// initialize a task temporal dir
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
index dfc8a9b..6c0af89 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java
@@ -379,12 +379,15 @@ public class TaskAttemptContext {
return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]);
}
- public long getUniqueKeyFromFragments() {
- List<FragmentProto> totalFragments = new ArrayList<FragmentProto>();
- for (List<FragmentProto> eachFragments : fragmentMap.values()) {
- totalFragments.addAll(eachFragments);
+ public String getUniqueKeyFromFragments() {
+ StringBuilder sb = new StringBuilder();
+ for (List<FragmentProto> fragments : fragmentMap.values()) {
+ for (FragmentProto f : fragments) {
+ FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, f);
+ sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength());
+ }
}
- return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()]));
+ return sb.toString();
}
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 4e4b710..615dbb9 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -36,11 +36,11 @@ import org.apache.tajo.engine.codegen.TajoClassLoader;
import org.apache.tajo.engine.function.FunctionLoader;
import org.apache.tajo.engine.json.CoreGsonHelper;
import org.apache.tajo.engine.parser.SQLAnalyzer;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.*;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.serder.EvalNodeDeserializer;
import org.apache.tajo.plan.serder.EvalNodeSerializer;
-import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.serder.PlanProto;
import org.apache.tajo.plan.verifier.LogicalPlanVerifier;
@@ -98,7 +98,7 @@ public class ExprTestBase {
analyzer = new SQLAnalyzer();
preLogicalPlanVerifier = new PreLogicalPlanVerifier(cat);
planner = new LogicalPlanner(cat);
- optimizer = new LogicalOptimizer(util.getConfiguration());
+ optimizer = new LogicalOptimizer(util.getConfiguration(), cat);
annotatedPlanVerifier = new LogicalPlanVerifier(util.getConfiguration(), cat);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index c9b52fd..3122b25 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -104,9 +104,9 @@ public class TestLogicalOptimizer {
catalog.createFunction(funcDesc);
sqlAnalyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(util.getConfiguration());
defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration());
+ optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index 996d736..0bf4602 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -497,7 +497,7 @@ public class TestLogicalPlanner {
Schema expected = tpch.getOutSchema("q2");
assertSchema(expected, node.getOutSchema());
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration());
+ LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
optimizer.optimize(plan);
LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.JOIN);
@@ -536,7 +536,7 @@ public class TestLogicalPlanner {
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration());
+ LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
optimizer.optimize(plan);
LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN);
@@ -577,7 +577,7 @@ public class TestLogicalPlanner {
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration());
+ LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
optimizer.optimize(plan);
LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN);
@@ -624,7 +624,7 @@ public class TestLogicalPlanner {
LogicalNode node = plan.getRootBlock().getRoot();
testJsonSerDerObject(node);
- LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration());
+ LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
optimizer.optimize(plan);
Map<BinaryEval, Boolean> scanMap = TUtil.newHashMap();
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
index 3803c7a..9f20776 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java
@@ -180,7 +180,7 @@ public class TestBroadcastJoinPlan {
"join small2 on small1_id = small2_id";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -241,7 +241,7 @@ public class TestBroadcastJoinPlan {
"join small3 on small1_id = small3_id";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -305,7 +305,7 @@ public class TestBroadcastJoinPlan {
"join large2 on large1_id = large2_id ";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -333,7 +333,7 @@ public class TestBroadcastJoinPlan {
"join small2 on large2_id = small2_id";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -383,7 +383,7 @@ public class TestBroadcastJoinPlan {
"join small2 on a.small1_id = small2_id";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -424,7 +424,7 @@ public class TestBroadcastJoinPlan {
"join (select * from small1) a on large1_id = a.small1_id";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -480,7 +480,7 @@ public class TestBroadcastJoinPlan {
"left outer join large2 on small1_id = large2_id ";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -534,7 +534,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on large1_id = small3_id ";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -617,7 +617,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on large3_id = small3_id ";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -700,7 +700,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on small1_id = small3_id ";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -759,7 +759,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on small1_id = small3_id " ;
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -812,7 +812,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on small1_id = small3_id " ;
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -904,7 +904,7 @@ public class TestBroadcastJoinPlan {
"left outer join small3 on small3_id = large1_id " ;
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
@@ -969,7 +969,7 @@ public class TestBroadcastJoinPlan {
"left outer join small2 on large1_id = small2_id " ;
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
deleted file mode 100644
index c897461..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.engine.planner.physical;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.tajo.LocalTajoTestingUtility;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.TajoTestingCluster;
-import org.apache.tajo.algebra.Expr;
-import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.plan.LogicalOptimizer;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.LogicalPlanner;
-import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
-import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.plan.logical.ScanNode;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.storage.*;
-import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.storage.fragment.FragmentConvertor;
-import org.apache.tajo.storage.index.bst.BSTIndex;
-import org.apache.tajo.util.CommonTestingUtil;
-import org.apache.tajo.worker.TaskAttemptContext;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Random;
-import java.util.Stack;
-
-import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
-import static org.junit.Assert.assertEquals;
-
-public class TestBSTIndexExec {
-
- private TajoConf conf;
- private Path idxPath;
- private CatalogService catalog;
- private SQLAnalyzer analyzer;
- private LogicalPlanner planner;
- private LogicalOptimizer optimizer;
- private FileStorageManager sm;
- private Schema idxSchema;
- private BaseTupleComparator comp;
- private BSTIndex.BSTIndexWriter writer;
- private HashMap<Integer , Integer> randomValues ;
- private int rndKey = -1;
- private FileSystem fs;
- private TableMeta meta;
- private Path tablePath;
-
- private Random rnd = new Random(System.currentTimeMillis());
-
- private TajoTestingCluster util;
-
- @Before
- public void setup() throws Exception {
- this.randomValues = new HashMap<Integer, Integer>();
- this.conf = new TajoConf();
- util = new TajoTestingCluster();
- util.startCatalogCluster();
- catalog = util.getMiniCatalogCluster().getCatalog();
-
- Path workDir = CommonTestingUtil.getTestDir();
- catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString());
- catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME);
- sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir);
-
- idxPath = new Path(workDir, "test.idx");
-
- Schema schema = new Schema();
- schema.addColumn("managerid", Type.INT4);
- schema.addColumn("empid", Type.INT4);
- schema.addColumn("deptname", Type.TEXT);
-
- this.idxSchema = new Schema();
- idxSchema.addColumn("managerid", Type.INT4);
- SortSpec[] sortKeys = new SortSpec[1];
- sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false);
- this.comp = new BaseTupleComparator(idxSchema, sortKeys);
-
- this.writer = new BSTIndex(conf).getIndexWriter(idxPath,
- BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp);
- writer.setLoadNum(100);
- writer.open();
- long offset;
-
- meta = CatalogUtil.newTableMeta(StoreType.CSV);
- tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv");
- fs = tablePath.getFileSystem(conf);
- fs.mkdirs(tablePath.getParent());
-
- FileAppender appender = (FileAppender)sm.getAppender(meta, schema, tablePath);
- appender.init();
- Tuple tuple = new VTuple(schema.size());
- for (int i = 0; i < 10000; i++) {
-
- Tuple key = new VTuple(this.idxSchema.size());
- int rndKey = rnd.nextInt(250);
- if(this.randomValues.containsKey(rndKey)) {
- int t = this.randomValues.remove(rndKey) + 1;
- this.randomValues.put(rndKey, t);
- } else {
- this.randomValues.put(rndKey, 1);
- }
-
- key.put(new Datum[] { DatumFactory.createInt4(rndKey) });
- tuple.put(new Datum[] { DatumFactory.createInt4(rndKey),
- DatumFactory.createInt4(rnd.nextInt(10)),
- DatumFactory.createText("dept_" + rnd.nextInt(10)) });
- offset = appender.getOffset();
- appender.addTuple(tuple);
- writer.write(key, offset);
- }
- appender.flush();
- appender.close();
- writer.close();
-
- TableDesc desc = new TableDesc(
- CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
- sm.getTablePath("employee").toUri());
- catalog.createTable(desc);
-
- analyzer = new SQLAnalyzer();
- planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
- }
-
- @After
- public void tearDown() {
- util.shutdownCatalogCluster();
- }
-
- @Test
- public void testEqual() throws Exception {
- this.rndKey = rnd.nextInt(250);
- final String QUERY = "select * from employee where managerId = " + rndKey;
-
- FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE);
- Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual");
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
- Expr expr = analyzer.parse(QUERY);
- LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr);
- LogicalNode rootNode = optimizer.optimize(plan);
-
- TmpPlanner phyPlanner = new TmpPlanner(conf);
- PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
-
- int tupleCount = this.randomValues.get(rndKey);
- int counter = 0;
- exec.init();
- while (exec.next() != null) {
- counter ++;
- }
- exec.close();
- assertEquals(tupleCount , counter);
- }
-
- private class TmpPlanner extends PhysicalPlannerImpl {
- public TmpPlanner(TajoConf conf) {
- super(conf);
- }
-
- @Override
- public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, Stack<LogicalNode> stack)
- throws IOException {
- Preconditions.checkNotNull(ctx.getTable(scanNode.getTableName()),
- "Error: There is no table matched to %s", scanNode.getTableName());
-
- List<FileFragment> fragments = FragmentConvertor.convert(ctx.getConf(), ctx.getTables(scanNode.getTableName()));
-
- Datum[] datum = new Datum[]{DatumFactory.createInt4(rndKey)};
-
- return new BSTIndexScanExec(ctx, scanNode, fragments.get(0), idxPath, idxSchema, comp , datum);
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index 64da88b..a64b525 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -63,6 +63,7 @@ public class TestHashAntiJoinExec {
private LogicalOptimizer optimizer;
private StorageManager sm;
private Path testDir;
+ private QueryContext queryContext;
private TableDesc employee;
private TableDesc people;
@@ -128,11 +129,12 @@ public class TestHashAntiJoinExec {
appender.flush();
appender.close();
+ queryContext = new QueryContext(conf);
people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath);
catalog.createTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
}
@After
@@ -159,7 +161,7 @@ public class TestHashAntiJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+ TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 4e218c5..196f3bf 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -64,6 +64,7 @@ public class TestHashSemiJoinExec {
private LogicalOptimizer optimizer;
private StorageManager sm;
private Path testDir;
+ private QueryContext queryContext;
private TableDesc employee;
private TableDesc people;
@@ -133,11 +134,12 @@ public class TestHashSemiJoinExec {
appender.flush();
appender.close();
+ queryContext = new QueryContext(conf);
people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath);
catalog.createTable(people);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
}
@After
@@ -164,7 +166,7 @@ public class TestHashSemiJoinExec {
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin");
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+ TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index a4e49f7..9de58a3 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -171,12 +171,13 @@ public class TestPhysicalPlanner {
}
appender.flush();
appender.close();
+
+ defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
catalog.createTable(score);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
- defaultContext = LocalTajoTestingUtility.createDummyContext(conf);
masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null);
createLargeScoreTable();
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 2e093c1..84abfff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -61,6 +61,7 @@ public class TestSortExec {
private static Path workDir;
private static Path tablePath;
private static TableMeta employeeMeta;
+ private static QueryContext queryContext;
private static Random rnd = new Random(System.currentTimeMillis());
@@ -101,9 +102,10 @@ public class TestSortExec {
tablePath.toUri());
catalog.createTable(desc);
+ queryContext = new QueryContext(conf);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
}
public static String[] QUERIES = {
@@ -113,7 +115,7 @@ public class TestSortExec {
public final void testNext() throws IOException, PlanningException {
FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec");
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
+ TaskAttemptContext ctx = new TaskAttemptContext(queryContext,
LocalTajoTestingUtility
.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir);
ctx.setEnforcer(new Enforcer());
[5/5] tajo git commit: TAJO-838: Improve query planner to utilize
index. (jihoon)
Posted by ji...@apache.org.
TAJO-838: Improve query planner to utilize index. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/071c5d05
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/071c5d05
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/071c5d05
Branch: refs/heads/index_support
Commit: 071c5d05dff157e47d1bb02e79de88a9577c3346
Parents: 1fad72e
Author: Jihoon Son <ji...@apache.org>
Authored: Fri Jan 9 01:13:52 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Fri Jan 9 01:13:52 2015 +0900
----------------------------------------------------------------------
.../java/org/apache/tajo/algebra/DropIndex.java | 54 ++++
.../java/org/apache/tajo/algebra/OpType.java | 1 +
.../tajo/catalog/AbstractCatalogClient.java | 120 +++++++--
.../src/main/proto/CatalogProtocol.proto | 6 +-
.../apache/tajo/catalog/CatalogConstants.java | 1 +
.../org/apache/tajo/catalog/CatalogService.java | 24 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 46 +++-
.../java/org/apache/tajo/catalog/IndexDesc.java | 132 +++++-----
.../java/org/apache/tajo/catalog/IndexMeta.java | 180 +++++++++++++
.../java/org/apache/tajo/catalog/TableDesc.java | 4 -
.../catalog/exception/NoSuchIndexException.java | 2 +-
.../src/main/proto/CatalogProtos.proto | 36 +--
.../org/apache/tajo/catalog/TestIndexDesc.java | 58 +++--
.../tajo/catalog/store/HCatalogStore.java | 62 +++--
.../org/apache/tajo/catalog/CatalogServer.java | 93 +++++--
.../dictionary/IndexesTableDescriptor.java | 8 +-
.../tajo/catalog/store/AbstractDBStore.java | 186 +++++++------
.../apache/tajo/catalog/store/CatalogStore.java | 29 +--
.../org/apache/tajo/catalog/store/MemStore.java | 120 +++++----
.../src/main/resources/schemas/derby/derby.xml | 27 +-
.../main/resources/schemas/mariadb/indexes.sql | 14 +-
.../main/resources/schemas/mysql/indexes.sql | 14 +-
.../main/resources/schemas/oracle/indexes.sql | 18 +-
.../resources/schemas/postgresql/postgresql.xml | 29 ++-
.../org/apache/tajo/catalog/TestCatalog.java | 76 +++---
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 12 +-
.../cli/tsql/commands/DescTableCommand.java | 19 ++
.../apache/tajo/client/CatalogAdminClient.java | 16 ++
.../tajo/client/CatalogAdminClientImpl.java | 138 +++++++++-
.../org/apache/tajo/client/QueryClientImpl.java | 40 +--
.../org/apache/tajo/client/QueryStatus.java | 8 +-
.../apache/tajo/client/SessionConnection.java | 12 +-
.../org/apache/tajo/client/TajoClientImpl.java | 37 +++
tajo-client/src/main/proto/ClientProtos.proto | 71 ++---
.../main/proto/TajoMasterClientProtocol.proto | 9 +
.../java/org/apache/tajo/OverridableConf.java | 2 +-
.../main/java/org/apache/tajo/SessionVars.java | 4 +
.../java/org/apache/tajo/conf/TajoConf.java | 6 +-
.../main/java/org/apache/tajo/util/TUtil.java | 10 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 9 +
.../engine/codegen/ExecutorPreCompiler.java | 7 +
.../apache/tajo/engine/parser/SQLAnalyzer.java | 8 +-
.../engine/planner/PhysicalPlannerImpl.java | 21 +-
.../engine/planner/global/GlobalPlanner.java | 9 +
.../planner/physical/BSTIndexScanExec.java | 96 +++++--
.../engine/planner/physical/ProjectionExec.java | 1 +
.../engine/planner/physical/StoreIndexExec.java | 13 +-
.../utils/test/ErrorInjectionRewriter.java | 5 +-
.../org/apache/tajo/master/GlobalEngine.java | 11 +-
.../NonForwardQueryResultSystemScanner.java | 46 ++--
.../tajo/master/TajoMasterClientService.java | 258 +++++++++++++++----
.../apache/tajo/master/exec/DDLExecutor.java | 109 +++++---
.../apache/tajo/master/exec/QueryExecutor.java | 59 ++++-
.../apache/tajo/master/querymaster/Query.java | 45 ++++
.../master/querymaster/QueryMasterTask.java | 23 +-
.../main/java/org/apache/tajo/util/IPCUtil.java | 44 ++++
.../java/org/apache/tajo/util/IndexUtil.java | 149 -----------
.../tajo/webapp/QueryExecutorServlet.java | 8 +-
.../tajo/worker/TajoWorkerClientService.java | 27 +-
.../main/java/org/apache/tajo/worker/Task.java | 29 +--
.../apache/tajo/worker/TaskAttemptContext.java | 13 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 4 +-
.../engine/planner/TestLogicalOptimizer.java | 2 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../planner/global/TestBroadcastJoinPlan.java | 28 +-
.../planner/physical/TestBSTIndexExec.java | 206 ---------------
.../planner/physical/TestHashAntiJoinExec.java | 6 +-
.../planner/physical/TestHashSemiJoinExec.java | 6 +-
.../planner/physical/TestPhysicalPlanner.java | 5 +-
.../engine/planner/physical/TestSortExec.java | 6 +-
.../tajo/engine/query/TestCreateIndex.java | 44 +++-
.../apache/tajo/engine/query/TestIndexScan.java | 119 +++++++++
.../tajo/engine/query/TestTablePartitions.java | 8 +-
.../tajo/master/TestExecutionBlockCursor.java | 2 +-
.../apache/tajo/master/TestGlobalPlanner.java | 2 +-
.../tajo/master/querymaster/TestKillQuery.java | 2 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 2 +-
.../testCreateIndexOnMultiExprs.sql | 1 +
.../TestIndexScan/testOnMultipleExprs.result | 3 +
.../TestIndexScan/testOnMultipleKeys.result | 3 +
.../TestIndexScan/testOnMultipleKeys2.result | 3 +
.../testOnSortedNonUniqueKeys.result | 4 +
.../TestIndexScan/testOnUnsortedTextKeys.result | 3 +
.../TestIndexScan/testWithGroupBy.result | 3 +
.../results/TestIndexScan/testWithJoin.result | 4 +
.../results/TestIndexScan/testWithSort.result | 4 +
.../TestTajoCli/testHelpSessionVars.result | 2 +
tajo-docs/src/main/sphinx/index.rst | 1 +
tajo-docs/src/main/sphinx/index/future_work.rst | 8 +
tajo-docs/src/main/sphinx/index/how_to_use.rst | 69 +++++
tajo-docs/src/main/sphinx/index/types.rst | 7 +
tajo-docs/src/main/sphinx/index_overview.rst | 20 ++
tajo-docs/src/main/sphinx/sql_language/ddl.rst | 33 ++-
.../org/apache/tajo/plan/LogicalOptimizer.java | 14 +-
.../java/org/apache/tajo/plan/LogicalPlan.java | 28 ++
.../tajo/plan/LogicalPlanPreprocessor.java | 8 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 27 +-
.../org/apache/tajo/plan/NamedExprsManager.java | 4 +-
.../tajo/plan/algebra/AlgebraVisitor.java | 1 +
.../tajo/plan/algebra/BaseAlgebraVisitor.java | 8 +
.../apache/tajo/plan/expr/AlgebraicUtil.java | 69 +++++
.../org/apache/tajo/plan/expr/EvalTreeUtil.java | 6 +-
.../tajo/plan/logical/CreateIndexNode.java | 89 ++++---
.../apache/tajo/plan/logical/DropIndexNode.java | 92 +++++++
.../apache/tajo/plan/logical/IndexScanNode.java | 88 +++----
.../org/apache/tajo/plan/logical/NodeType.java | 3 +-
.../rewrite/BaseLogicalPlanRewriteEngine.java | 10 +-
.../BaseLogicalPlanRewriteRuleProvider.java | 4 +-
.../plan/rewrite/LogicalPlanRewriteEngine.java | 3 +-
.../plan/rewrite/LogicalPlanRewriteRule.java | 5 +-
.../rewrite/LogicalPlanRewriteRuleContext.java | 65 +++++
.../tajo/plan/rewrite/rules/AccessPathInfo.java | 52 ++++
.../plan/rewrite/rules/AccessPathRewriter.java | 129 ++++++++++
.../plan/rewrite/rules/FilterPushDownRule.java | 98 ++++++-
.../tajo/plan/rewrite/rules/IndexScanInfo.java | 113 ++++++++
.../rules/LogicalPlanEqualityTester.java | 8 +-
.../rewrite/rules/PartitionedTableRewriter.java | 10 +-
.../rewrite/rules/ProjectionPushDownRule.java | 16 +-
.../tajo/plan/rewrite/rules/SeqScanInfo.java | 43 ++++
.../plan/serder/LogicalNodeDeserializer.java | 71 ++++-
.../tajo/plan/serder/LogicalNodeSerializer.java | 74 ++++++
.../org/apache/tajo/plan/util/IndexUtil.java | 72 ++++++
.../org/apache/tajo/plan/util/PlannerUtil.java | 35 ++-
.../plan/visitor/BasicLogicalPlanVisitor.java | 18 ++
.../plan/visitor/ExplainLogicalPlanVisitor.java | 7 +
.../tajo/plan/visitor/LogicalPlanVisitor.java | 6 +
tajo-plan/src/main/proto/Plan.proto | 85 ++++--
.../storage/hbase/AddSortForInsertRewriter.java | 8 +-
128 files changed, 3414 insertions(+), 1287 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
new file mode 100644
index 0000000..5a75e78
--- /dev/null
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.algebra;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+
+public class DropIndex extends Expr {
+ @Expose @SerializedName("IndexName")
+ private String indexName;
+
+ public DropIndex(final String indexName) {
+ super(OpType.DropIndex);
+ this.indexName = indexName;
+ }
+
+ @Override
+ public int hashCode() {
+ return indexName.hashCode();
+ }
+
+ @Override
+ boolean equalsTo(Expr expr) {
+ DropIndex other = (DropIndex) expr;
+ return this.indexName.equals(other.indexName);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ DropIndex clone = (DropIndex) super.clone();
+ clone.indexName = indexName;
+ return clone;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
index 32c51db..47fea64 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java
@@ -54,6 +54,7 @@ public enum OpType {
AlterTablespace(AlterTablespace.class),
AlterTable(AlterTable.class),
CreateIndex(CreateIndex.class),
+ DropIndex(DropIndex.class),
TruncateTable(TruncateTable.class),
// Insert or Update
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index 8ef1c9a..dc12614 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -43,6 +43,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.ProtoUtil;
+import org.apache.tajo.util.TUtil;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -379,6 +380,24 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
+ public List<IndexDescProto> getAllIndexes() {
+ try {
+ return new ServerCallable<List<IndexDescProto>>(pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
+
+ @Override
+ public List<IndexDescProto> call(NettyClientBase client) throws Exception {
+ CatalogProtocolService.BlockingInterface stub = getStub(client);
+ GetIndexesProto response = stub.getAllIndexes(null, ProtoUtil.NULL_PROTO);
+ return response.getIndexList();
+ }
+ }.withRetries();
+ } catch (ServiceException e) {
+ LOG.error(e.getMessage(), e);
+ return null;
+ }
+ }
+
+ @Override
public final PartitionMethodDesc getPartitionMethod(final String databaseName, final String tableName) {
try {
return new ServerCallable<PartitionMethodDesc>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
@@ -582,17 +601,40 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
- public boolean existIndexByColumn(final String databaseName, final String tableName, final String columnName) {
+ public boolean existIndexByColumns(final String databaseName, final String tableName, final Column [] columns) {
+ return existIndexByColumnNames(databaseName, tableName, extractColumnNames(columns));
+ }
+
+ @Override
+ public boolean existIndexByColumnNames(final String databaseName, final String tableName, final String [] columnNames) {
try {
return new ServerCallable<Boolean>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
public Boolean call(NettyClientBase client) throws ServiceException {
- GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder();
+ GetIndexByColumnNamesRequest.Builder builder = GetIndexByColumnNamesRequest.newBuilder();
builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
- builder.setColumnName(columnName);
+ for (String colunName : columnNames) {
+ builder.addColumnNames(colunName);
+ }
CatalogProtocolService.BlockingInterface stub = getStub(client);
- return stub.existIndexByColumn(null, builder.build()).getValue();
+ return stub.existIndexByColumnNames(null, builder.build()).getValue();
+ }
+ }.withRetries();
+ } catch (ServiceException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean existIndexesByTable(final String databaseName, final String tableName) {
+ try {
+ return new ServerCallable<Boolean>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
+ public Boolean call(NettyClientBase client) throws ServiceException {
+
+ CatalogProtocolService.BlockingInterface stub = getStub(client);
+ return stub.existIndexesByTable(null, CatalogUtil.buildTableIdentifier(databaseName, tableName)).getValue();
}
}.withRetries();
} catch (ServiceException e) {
@@ -621,20 +663,60 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
}
+ private static String[] extractColumnNames(Column[] columns) {
+ String[] columnNames = new String [columns.length];
+ for (int i = 0; i < columnNames.length; i++) {
+ columnNames[i] = columns[i].getSimpleName();
+ }
+ return columnNames;
+ }
+
+ @Override
+ public final IndexDesc getIndexByColumns(final String databaseName,
+ final String tableName,
+ final Column [] columns) {
+ return getIndexByColumnNames(databaseName, tableName, extractColumnNames(columns));
+ }
+
@Override
- public final IndexDesc getIndexByColumn(final String databaseName,
- final String tableName,
- final String columnName) {
+ public final IndexDesc getIndexByColumnNames(final String databaseName,
+ final String tableName,
+ final String [] columnNames) {
try {
return new ServerCallable<IndexDesc>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
public IndexDesc call(NettyClientBase client) throws ServiceException {
- GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder();
+ GetIndexByColumnNamesRequest.Builder builder = GetIndexByColumnNamesRequest.newBuilder();
builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
- builder.setColumnName(columnName);
+ for (String columnName : columnNames) {
+ builder.addColumnNames(columnName);
+ }
CatalogProtocolService.BlockingInterface stub = getStub(client);
- return new IndexDesc(stub.getIndexByColumn(null, builder.build()));
+ return new IndexDesc(stub.getIndexByColumnNames(null, builder.build()));
+ }
+ }.withRetries();
+ } catch (ServiceException e) {
+ LOG.error(e.getMessage(), e);
+ return null;
+ }
+ }
+
+ @Override
+ public final Collection<IndexDesc> getAllIndexesByTable(final String databaseName,
+ final String tableName) {
+ try {
+ return new ServerCallable<Collection<IndexDesc>>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
+ @Override
+ public Collection<IndexDesc> call(NettyClientBase client) throws Exception {
+ TableIdentifierProto proto = CatalogUtil.buildTableIdentifier(databaseName, tableName);
+ CatalogProtocolService.BlockingInterface stub = getStub(client);
+ GetAllIndexesResponse response = stub.getAllIndexesByTable(null, proto);
+ List<IndexDesc> indexDescs = TUtil.newList();
+ for (IndexDescProto descProto : response.getIndexDescList()) {
+ indexDescs.add(new IndexDesc(descProto));
+ }
+ return indexDescs;
}
}.withRetries();
} catch (ServiceException e) {
@@ -665,24 +747,6 @@ public abstract class AbstractCatalogClient implements CatalogService {
}
@Override
- public List<IndexProto> getAllIndexes() {
- try {
- return new ServerCallable<List<IndexProto>>(pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
-
- @Override
- public List<IndexProto> call(NettyClientBase client) throws Exception {
- CatalogProtocolService.BlockingInterface stub = getStub(client);
- GetIndexesProto response = stub.getAllIndexes(null, ProtoUtil.NULL_PROTO);
- return response.getIndexList();
- }
- }.withRetries();
- } catch (ServiceException e) {
- LOG.error(e.getMessage(), e);
- return null;
- }
- }
-
- @Override
public final boolean createFunction(final FunctionDesc funcDesc) {
try {
return new ServerCallable<Boolean>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
index cae5d88..cbe689f 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
+++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto
@@ -66,9 +66,11 @@ service CatalogProtocolService {
rpc createIndex(IndexDescProto) returns (BoolProto);
rpc dropIndex(IndexNameProto) returns (BoolProto);
rpc existIndexByName(IndexNameProto) returns (BoolProto);
- rpc existIndexByColumn(GetIndexByColumnRequest) returns (BoolProto);
+ rpc existIndexByColumnNames(GetIndexByColumnNamesRequest) returns (BoolProto);
+ rpc existIndexesByTable(TableIdentifierProto) returns (BoolProto);
rpc getIndexByName(IndexNameProto) returns (IndexDescProto);
- rpc getIndexByColumn(GetIndexByColumnRequest) returns (IndexDescProto);
+ rpc getIndexByColumnNames(GetIndexByColumnNamesRequest) returns (IndexDescProto);
+ rpc getAllIndexesByTable(TableIdentifierProto) returns (GetAllIndexesResponse);
rpc getAllIndexes(NullProto) returns (GetIndexesProto);
rpc createFunction(FunctionDescProto) returns (BoolProto);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
index 6ec52b9..c7df801 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java
@@ -52,5 +52,6 @@ public class CatalogConstants {
public static final String COL_TABLESPACE_PK = "SPACE_ID";
public static final String COL_DATABASES_PK = "DB_ID";
public static final String COL_TABLES_PK = "TID";
+ public static final String COL_INDEXES_PK = "INDEX_ID";
public static final String COL_TABLES_NAME = "TABLE_NAME";
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
index 2a5d890..dd26a27 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java
@@ -19,13 +19,7 @@
package org.apache.tajo.catalog;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes.DataType;
import java.util.Collection;
@@ -154,6 +148,8 @@ public interface CatalogService {
*/
List<ColumnProto> getAllColumns();
+ List<IndexDescProto> getAllIndexes();
+
/**
*
* @return All FunctionDescs
@@ -190,15 +186,21 @@ public interface CatalogService {
boolean existIndexByName(String databaseName, String indexName);
- boolean existIndexByColumn(String databaseName, String tableName, String columnName);
+ boolean existIndexByColumns(String databaseName, String tableName, Column[] columns);
+
+ boolean existIndexByColumnNames(String databaseName, String tableName, String [] columnNames);
+
+ boolean existIndexesByTable(String databaseName, String tableName);
IndexDesc getIndexByName(String databaseName, String indexName);
- IndexDesc getIndexByColumn(String databaseName, String tableName, String columnName);
+ IndexDesc getIndexByColumns(String databaseName, String tableName, Column [] columns);
+
+ IndexDesc getIndexByColumnNames(String databaseName, String tableName, String [] columnNames);
+
+ Collection<IndexDesc> getAllIndexesByTable(String databaseName, String tableName);
boolean dropIndex(String databaseName, String indexName);
-
- List<IndexProto> getAllIndexes();
boolean createFunction(FunctionDesc funcDesc);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index bb15ec1..a593cd9 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
@@ -39,12 +40,8 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
-import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.common.TajoDataTypes.Type;
public class CatalogUtil {
@@ -868,4 +865,43 @@ public class CatalogUtil {
return options;
}
+
+ /**
+ * Make a unique name by concatenating column names.
+ * The concatenation is performed in sequence of columns' occurrence in the relation schema.
+ *
+ * @param originalSchema original relation schema
+ * @param columnNames column names which will be unified
+ * @return unified name
+ */
+ public static String getUnifiedSimpleColumnName(Schema originalSchema, String[] columnNames) {
+ String[] simpleNames = new String[columnNames.length];
+ for (int i = 0; i < simpleNames.length; i++) {
+ String[] identifiers = columnNames[i].split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP);
+ simpleNames[i] = identifiers[identifiers.length-1];
+ }
+ Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema));
+ StringBuilder sb = new StringBuilder();
+ for (String colName : simpleNames) {
+ sb.append(colName).append("_");
+ }
+ sb.deleteCharAt(sb.length()-1);
+ return sb.toString();
+ }
+
+ /**
+ * Given column names, compare the position of columns in the relation schema.
+ */
+ public static class ColumnPosComparator implements Comparator<String> {
+
+ private Schema originlSchema;
+ public ColumnPosComparator(Schema originalSchema) {
+ this.originlSchema = originalSchema;
+ }
+
+ @Override
+ public int compare(String o1, String o2) {
+ return originlSchema.getColumnId(o1) - originlSchema.getColumnId(o2);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
index 7dca4eb..9f64913 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java
@@ -21,80 +21,91 @@ package org.apache.tajo.catalog;
import com.google.common.base.Objects;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.common.ProtoObject;
-public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
- private IndexDescProto.Builder builder;
+import java.net.URI;
+import java.net.URISyntaxException;
- private String name;
- private Path indexPath; // required
+public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
private String databaseName; // required
private String tableName; // required
- private Column column; // required
- private IndexMethod indexMethod; // required
- private boolean isUnique = false; // optional [default = false]
- private boolean isClustered = false; // optional [default = false]
- private boolean isAscending = false; // optional [default = false]
-
+ private IndexMeta indexMeta; // required
+
public IndexDesc() {
}
- public IndexDesc(String name, Path indexPath, String databaseName, String tableName, Column column,
- IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) {
+ public IndexDesc(String databaseName, String tableName, String indexName, URI indexPath, SortSpec[] keySortSpecs,
+ IndexMethod type, boolean isUnique, boolean isClustered, Schema targetRelationSchema) {
this();
- this.name = name;
- this.indexPath = indexPath;
- this.databaseName = databaseName;
- this.tableName = tableName;
- this.column = column;
- this.indexMethod = type;
- this.isUnique = isUnique;
- this.isClustered = isClustered;
- this.isAscending = isAscending;
+ this.set(databaseName, tableName, indexName, indexPath, keySortSpecs, type, isUnique, isClustered,
+ targetRelationSchema);
}
public IndexDesc(IndexDescProto proto) {
- this(proto.getName(), new Path(proto.getIndexPath()),
- proto.getTableIdentifier().getDatabaseName(),
- proto.getTableIdentifier().getTableName(),
- new Column(proto.getColumn()),
- proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending());
+ this();
+
+ SortSpec[] keySortSpecs = new SortSpec[proto.getKeySortSpecsCount()];
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ keySortSpecs[i] = new SortSpec(proto.getKeySortSpecs(i));
+ }
+
+ try {
+ this.set(proto.getTableIdentifier().getDatabaseName(),
+ proto.getTableIdentifier().getTableName(),
+ proto.getIndexName(), new URI(proto.getIndexPath()),
+ keySortSpecs,
+ proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(),
+ new Schema(proto.getTargetRelationSchema()));
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
}
- public String getName() {
- return name;
+ public void set(String databaseName, String tableName, String indexName, URI indexPath, SortSpec[] keySortSpecs,
+ IndexMethod type, boolean isUnique, boolean isClustered, Schema targetRelationSchema) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.indexMeta = new IndexMeta(indexName, indexPath, keySortSpecs, type, isUnique, isClustered,
+ targetRelationSchema);
}
-
- public Path getIndexPath() {
- return indexPath;
+
+ public String getDatabaseName() {
+ return databaseName;
}
-
+
public String getTableName() {
return tableName;
}
+
+ public String getName() {
+ return indexMeta.getIndexName();
+ }
- public Column getColumn() {
- return column;
+ public URI getIndexPath() {
+ return indexMeta.getIndexPath();
+ }
+
+ public SortSpec[] getKeySortSpecs() {
+ return indexMeta.getKeySortSpecs();
}
public IndexMethod getIndexMethod() {
- return this.indexMethod;
+ return indexMeta.getIndexMethod();
}
public boolean isClustered() {
- return this.isClustered;
+ return indexMeta.isClustered();
}
public boolean isUnique() {
- return this.isUnique;
+ return indexMeta.isUnique();
}
-
- public boolean isAscending() {
- return this.isAscending;
+
+ public Schema getTargetRelationSchema() {
+ return indexMeta.getTargetRelationSchema();
}
@Override
@@ -110,13 +121,15 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
}
builder.setTableIdentifier(tableIdentifierBuilder.build());
- builder.setName(this.name);
- builder.setIndexPath(this.indexPath.toString());
- builder.setColumn(this.column.getProto());
- builder.setIndexMethod(indexMethod);
- builder.setIsUnique(this.isUnique);
- builder.setIsClustered(this.isClustered);
- builder.setIsAscending(this.isAscending);
+ builder.setIndexName(indexMeta.getIndexName());
+ builder.setIndexPath(indexMeta.getIndexPath().toString());
+ for (SortSpec colSpec : indexMeta.getKeySortSpecs()) {
+ builder.addKeySortSpecs(colSpec.getProto());
+ }
+ builder.setIndexMethod(indexMeta.getIndexMethod());
+ builder.setIsUnique(indexMeta.isUnique());
+ builder.setIsClustered(indexMeta.isClustered());
+ builder.setTargetRelationSchema(indexMeta.getTargetRelationSchema().getProto());
return builder.build();
}
@@ -124,34 +137,23 @@ public class IndexDesc implements ProtoObject<IndexDescProto>, Cloneable {
public boolean equals(Object obj) {
if (obj instanceof IndexDesc) {
IndexDesc other = (IndexDesc) obj;
- return getIndexPath().equals(other.getIndexPath())
- && getName().equals(other.getName())
+ return getDatabaseName().equals(other.getDatabaseName())
&& getTableName().equals(other.getTableName())
- && getColumn().equals(other.getColumn())
- && getIndexMethod().equals(other.getIndexMethod())
- && isUnique() == other.isUnique()
- && isClustered() == other.isClustered()
- && isAscending() == other.isAscending();
+ && this.indexMeta.equals(other.indexMeta);
} else {
return false;
}
}
public int hashCode() {
- return Objects.hashCode(getName(), getIndexPath(), getTableName(), getColumn(),
- getIndexMethod(), isUnique(), isClustered(), isAscending());
+ return Objects.hashCode(databaseName, tableName, indexMeta);
}
public Object clone() throws CloneNotSupportedException {
IndexDesc desc = (IndexDesc) super.clone();
- desc.name = name;
- desc.indexPath = indexPath;
- desc.tableName = tableName;
- desc.column = column;
- desc.indexMethod = indexMethod;
- desc.isUnique = isUnique;
- desc.isClustered = isClustered;
- desc.isAscending = isAscending;
+ desc.databaseName = this.databaseName;
+ desc.tableName = this.tableName;
+ desc.indexMeta = (IndexMeta) this.indexMeta.clone();
return desc;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java
new file mode 100644
index 0000000..a911055
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog;
+
+import com.google.common.base.Objects;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
+import org.apache.tajo.util.KeyValueSet;
+import org.apache.tajo.util.TUtil;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Comparator;
+
+/**
+ * IndexMeta contains meta information of an index.
+ * Meta information is the name, an index method, a path to the stored location, index keys, and so on.
+ */
+public class IndexMeta implements Cloneable {
+ @Expose private String indexName; // index name
+ @Expose private IndexMethod indexMethod; // index method
+ @Expose private URI indexPath; // path to the location
+ @Expose private SortSpec[] keySortSpecs; // index keys. This array should always be sorted
+ // according to the position in the targetRelationSchema
+ @Expose private boolean isUnique = false; // unique key or not
+ @Expose private boolean isClustered = false; // clustered index or not
+ @Expose private Schema targetRelationSchema; // schema of the indexed relation
+ @Expose private KeyValueSet options; // index options. TODO: will be added
+
+ public IndexMeta() {}
+
+ public IndexMeta(String indexName, URI indexPath, SortSpec[] keySortSpecs,
+ IndexMethod type, boolean isUnique, boolean isClustered,
+ Schema targetRelationSchema) {
+ this.indexName = indexName;
+ this.indexPath = indexPath;
+ this.indexMethod = type;
+ this.isUnique = isUnique;
+ this.isClustered = isClustered;
+ this.targetRelationSchema = targetRelationSchema;
+ initKeySortSpecs(targetRelationSchema, keySortSpecs);
+ }
+
+ private void initKeySortSpecs(final Schema targetRelationSchema, final SortSpec[] keySortSpecs) {
+ this.targetRelationSchema = targetRelationSchema;
+ this.keySortSpecs = new SortSpec[keySortSpecs.length];
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ this.keySortSpecs[i] = new SortSpec(keySortSpecs[i].getSortKey(), keySortSpecs[i].isAscending(),
+ keySortSpecs[i].isNullFirst());
+ }
+ Arrays.sort(this.keySortSpecs, new Comparator<SortSpec>() {
+ @Override
+ public int compare(SortSpec o1, SortSpec o2) {
+ return targetRelationSchema.getColumnId(o1.getSortKey().getSimpleName())
+ - targetRelationSchema.getColumnId(o2.getSortKey().getSimpleName());
+ }
+ });
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public void setIndexName(final String indexName) {
+ this.indexName = indexName;
+ }
+
+ public IndexMethod getIndexMethod() {
+ return indexMethod;
+ }
+
+ public void setIndexMethod(final IndexMethod type) {
+ this.indexMethod = type;
+ }
+
+ public URI getIndexPath() {
+ return indexPath;
+ }
+
+ public void setIndexPath(final URI indexPath) {
+ this.indexPath = indexPath;
+ }
+
+ public SortSpec[] getKeySortSpecs() {
+ return keySortSpecs;
+ }
+
+ public void setKeySortSpecs(final Schema targetRelationSchema, final SortSpec[] keySortSpecs) {
+ initKeySortSpecs(targetRelationSchema, keySortSpecs);
+ }
+
+ public boolean isUnique() {
+ return isUnique;
+ }
+
+ public void setUnique(boolean unique) {
+ this.isUnique = unique;
+ }
+
+ public boolean isClustered() {
+ return isClustered;
+ }
+
+ public void setClustered(boolean clustered) {
+ this.isClustered = clustered;
+ }
+
+ public Schema getTargetRelationSchema() {
+ return targetRelationSchema;
+ }
+
+ public KeyValueSet getOptions() {
+ return options;
+ }
+
+ public void setOptions(KeyValueSet options) {
+ this.options = options;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof IndexMeta) {
+ IndexMeta other = (IndexMeta) o;
+ return this.indexName.equals(other.indexName)
+ && this.indexPath.equals(other.indexPath)
+ && this.indexMethod.equals(other.indexMethod)
+ && TUtil.checkEquals(this.keySortSpecs, other.keySortSpecs)
+ && this.isUnique == other.isUnique
+ && this.isClustered == other.isClustered
+ && this.targetRelationSchema.equals(other.targetRelationSchema);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(indexName, indexPath, indexMethod, Objects.hashCode(keySortSpecs),
+ isUnique, isClustered, targetRelationSchema);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ IndexMeta clone = (IndexMeta) super.clone();
+ clone.indexName = indexName;
+ clone.indexPath = indexPath;
+ clone.indexMethod = indexMethod;
+ clone.keySortSpecs = new SortSpec[keySortSpecs.length];
+ for (int i = 0; i < keySortSpecs.length; i++) {
+ clone.keySortSpecs[i] = new SortSpec(this.keySortSpecs[i].getProto());
+ }
+ clone.isUnique = this.isUnique;
+ clone.isClustered = this.isClustered;
+ clone.targetRelationSchema = this.targetRelationSchema;
+ return clone;
+ }
+
+ @Override
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ return gson.toJson(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index ec679f9..f3f71ba 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -22,8 +22,6 @@ import com.google.common.base.Objects;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.Expose;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -37,8 +35,6 @@ import org.apache.tajo.util.TUtil;
import java.net.URI;
public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Cloneable {
- private final Log LOG = LogFactory.getLog(TableDesc.class);
-
@Expose protected String tableName; // required
@Expose protected Schema schema;
@Expose protected TableMeta meta; // required
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java
index 0bb7e32..71e6f15 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java
@@ -24,7 +24,7 @@ public class NoSuchIndexException extends CatalogException {
public NoSuchIndexException() {
}
- public NoSuchIndexException(String databaseName, String columnName) {
+ public NoSuchIndexException(String databaseName, String [] columnName) {
super(String.format("ERROR: index \" %s \" in %s does not exist", columnName, databaseName));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 6baead9..f10aa42 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -115,13 +115,13 @@ message NamespaceProto {
message IndexDescProto {
required TableIdentifierProto tableIdentifier = 1;
- required string name = 2;
- required string indexPath = 3;
- required ColumnProto column = 4;
- required IndexMethod indexMethod = 5;
- optional bool isUnique = 6 [default = false];
- optional bool isClustered = 7 [default = false];
- optional bool isAscending = 8 [default = false];
+ required string indexName = 2;
+ required IndexMethod indexMethod = 3;
+ required string indexPath = 4;
+ repeated SortSpecProto key_sort_specs = 5;
+ required SchemaProto targetRelationSchema = 6;
+ optional bool isUnique = 7 [default = false];
+ optional bool isClustered = 8 [default = false];
}
enum IndexMethod {
@@ -152,7 +152,7 @@ message GetColumnsProto {
}
message GetIndexesProto {
- repeated IndexProto index = 1;
+ repeated IndexDescProto index = 1;
}
message GetTableOptionsProto {
@@ -167,18 +167,6 @@ message GetTablePartitionsProto {
repeated TablePartitionProto part = 1;
}
-message IndexProto {
- required int32 dbId = 1;
- required int32 tId = 2;
- required string indexName = 3;
- required string columnName = 4;
- required string dataType = 5;
- required string indexType = 6;
- optional bool isUnique = 7 [default = false];
- optional bool isClustered = 8 [default = false];
- optional bool isAscending = 9 [default = false];
-}
-
message TableOptionProto {
required int32 tid = 1;
required KeyValueProto keyval = 2;
@@ -192,9 +180,9 @@ message TablePartitionProto {
optional string path = 5;
}
-message GetIndexByColumnRequest {
+message GetIndexByColumnNamesRequest {
required TableIdentifierProto tableIdentifier = 1;
- required string columnName = 2;
+ repeated string columnNames = 2;
}
message IndexNameProto {
@@ -203,6 +191,10 @@ message IndexNameProto {
required string indexName = 3;
}
+message GetAllIndexesResponse {
+ repeated IndexDescProto indexDesc = 1;
+}
+
message GetFunctionsResponse {
repeated FunctionDescProto functionDesc = 1;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
index afd88f1..247cd41 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java
@@ -18,7 +18,6 @@
package org.apache.tajo.catalog;
-import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -26,6 +25,9 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.net.URI;
+import java.net.URISyntaxException;
+
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
@@ -34,23 +36,29 @@ public class TestIndexDesc {
static IndexDesc desc1;
static IndexDesc desc2;
static IndexDesc desc3;
-
- static {
- desc1 = new IndexDesc(
- "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
-
- desc2 = new IndexDesc(
- "idx_test2", new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8),
- IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false);
-
- desc3 = new IndexDesc(
- "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
- }
+ static Schema relationSchema;
@BeforeClass
public static void setUp() throws Exception {
+ relationSchema = new Schema(new Column[]{new Column("id", Type.INT4),
+ new Column("score", Type.FLOAT8), new Column("name", Type.TEXT)});
+ SortSpec[] colSpecs1 = new SortSpec[1];
+ colSpecs1[0] = new SortSpec(new Column("id", Type.INT4), true, true);
+ desc1 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test", new URI("idx_test"), colSpecs1,
+ IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema);
+
+ SortSpec[] colSpecs2 = new SortSpec[1];
+ colSpecs2[0] = new SortSpec(new Column("score", Type.FLOAT8), false, false);
+ desc2 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test2", new URI("idx_test2"), colSpecs2,
+ IndexMethod.TWO_LEVEL_BIN_TREE, false, false, relationSchema);
+
+ SortSpec[] colSpecs3 = new SortSpec[1];
+ colSpecs3[0] = new SortSpec(new Column("id", Type.INT4), true, true);
+ desc3 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test", new URI("idx_test"), colSpecs3,
+ IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema);
}
@AfterClass
@@ -65,24 +73,28 @@ public class TestIndexDesc {
}
@Test
- public void testGetFields() {
+ public void testGetFields() throws URISyntaxException {
assertEquals("idx_test", desc1.getName());
assertEquals("indexed", desc1.getTableName());
- assertEquals(new Column("id", Type.INT4), desc1.getColumn());
+ assertEquals(1, desc1.getKeySortSpecs().length);
+ assertEquals(new Column("id", Type.INT4), desc1.getKeySortSpecs()[0].getSortKey());
+ assertEquals(true, desc1.getKeySortSpecs()[0].isAscending());
+ assertEquals(true, desc1.getKeySortSpecs()[0].isNullFirst());
assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod());
- assertEquals(new Path("idx_test"), desc1.getIndexPath());
+ assertEquals(new URI("idx_test"), desc1.getIndexPath());
assertEquals(true, desc1.isUnique());
assertEquals(true, desc1.isClustered());
- assertEquals(true, desc1.isAscending());
-
+
assertEquals("idx_test2", desc2.getName());
assertEquals("indexed", desc2.getTableName());
- assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn());
+ assertEquals(1, desc2.getKeySortSpecs().length);
+ assertEquals(new Column("score", Type.FLOAT8), desc2.getKeySortSpecs()[0].getSortKey());
+ assertEquals(false, desc2.getKeySortSpecs()[0].isAscending());
+ assertEquals(false, desc2.getKeySortSpecs()[0].isNullFirst());
assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod());
- assertEquals(new Path("idx_test2"), desc2.getIndexPath());
+ assertEquals(new URI("idx_test2"), desc2.getIndexPath());
assertEquals(false, desc2.isUnique());
assertEquals(false, desc2.isClustered());
- assertEquals(false, desc2.isAscending());
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index 2c3fc6a..78fee1d 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -43,7 +43,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto;
@@ -704,115 +704,135 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
@Override
public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMethodProto) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName)
throws CatalogException {
- return null; // TODO - not implemented yet
+ // TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public boolean existPartitionMethod(String databaseName, String tableName) throws CatalogException {
- return false; // TODO - not implemented yet
+ // TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public void dropPartitionMethod(String databaseName, String tableName) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public void addPartitions(CatalogProtos.PartitionsProto partitionsProto) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto partitionDescProto) throws CatalogException {
-
+ throw new UnsupportedOperationException();
}
@Override
public CatalogProtos.PartitionsProto getPartitions(String tableName) throws CatalogException {
- return null; // TODO - not implemented yet
+ // TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public CatalogProtos.PartitionDescProto getPartition(String partitionName) throws CatalogException {
- return null; // TODO - not implemented yet
+ // TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public void delPartition(String partitionName) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public void dropPartitions(String tableName) throws CatalogException {
-
+ throw new UnsupportedOperationException();
}
@Override
public final void addFunction(final FunctionDesc func) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public final void deleteFunction(final FunctionDesc func) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public final void existFunction(final FunctionDesc func) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
public final List<String> getAllFunctionNames() throws CatalogException {
// TODO - not implemented yet
- return null;
+ throw new UnsupportedOperationException();
}
@Override
- public void dropIndex(String databaseName, String indexName) throws CatalogException {
+ public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
- public boolean existIndexByName(String databaseName, String indexName) throws CatalogException {
+ public void dropIndex(String databaseName, String indexName) throws CatalogException {
// TODO - not implemented yet
- return false;
+ throw new UnsupportedOperationException();
}
@Override
- public CatalogProtos.IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException {
+ public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException {
// TODO - not implemented yet
- return null;
+ throw new UnsupportedOperationException();
}
@Override
- public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException {
+ public CatalogProtos.IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames)
+ throws CatalogException {
// TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
- public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException {
+ public boolean existIndexByName(String databaseName, String indexName) throws CatalogException {
// TODO - not implemented yet
- return null;
+ throw new UnsupportedOperationException();
}
@Override
- public CatalogProtos.IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName)
+ public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames)
throws CatalogException {
// TODO - not implemented yet
- return null;
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException {
+ // TODO - not implemented yet
+ throw new UnsupportedOperationException();
}
@Override
- public boolean existIndexByColumn(String databaseName, String tableName, String columnName) throws CatalogException {
+ public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException {
// TODO - not implemented yet
- return false;
+ throw new UnsupportedOperationException();
}
@Override
@@ -860,7 +880,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
@Override
- public List<IndexProto> getAllIndexes() throws CatalogException {
+ public List<IndexDescProto> getAllIndexes() throws CatalogException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 3f4d38d..a2b8eaf 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -870,12 +870,12 @@ public class CatalogServer extends AbstractService {
try {
if (store.existIndexByName(
databaseName,
- indexDesc.getName())) {
- throw new AlreadyExistsIndexException(indexDesc.getName());
+ indexDesc.getIndexName())) {
+ throw new AlreadyExistsIndexException(indexDesc.getIndexName());
}
store.createIndex(indexDesc);
} catch (Exception e) {
- LOG.error("ERROR : cannot add index " + indexDesc.getName(), e);
+ LOG.error("ERROR : cannot add index " + indexDesc.getIndexName(), e);
LOG.error(indexDesc);
throw new ServiceException(e);
} finally {
@@ -903,17 +903,35 @@ public class CatalogServer extends AbstractService {
}
@Override
- public BoolProto existIndexByColumn(RpcController controller, GetIndexByColumnRequest request)
+ public BoolProto existIndexByColumnNames(RpcController controller, GetIndexByColumnNamesRequest request)
throws ServiceException {
TableIdentifierProto identifier = request.getTableIdentifier();
String databaseName = identifier.getDatabaseName();
String tableName = identifier.getTableName();
- String columnName = request.getColumnName();
+ List<String> columnNames = request.getColumnNamesList();
rlock.lock();
try {
- return store.existIndexByColumn(databaseName, tableName, columnName) ?
+ return store.existIndexByColumns(databaseName, tableName,
+ columnNames.toArray(new String[columnNames.size()])) ?
+ ProtoUtil.TRUE : ProtoUtil.FALSE;
+ } catch (Exception e) {
+ LOG.error(e);
+ return BoolProto.newBuilder().setValue(false).build();
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ @Override
+ public BoolProto existIndexesByTable(RpcController controller, TableIdentifierProto request) throws ServiceException {
+ String databaseName = request.getDatabaseName();
+ String tableName = request.getTableName();
+
+ rlock.lock();
+ try {
+ return store.existIndexesByTable(databaseName, tableName) ?
ProtoUtil.TRUE : ProtoUtil.FALSE;
} catch (Exception e) {
LOG.error(e);
@@ -933,7 +951,7 @@ public class CatalogServer extends AbstractService {
rlock.lock();
try {
if (!store.existIndexByName(databaseName, indexName)) {
- throw new NoSuchIndexException(databaseName, indexName);
+ throw new NoSuchIndexException(indexName);
}
return store.getIndexByName(databaseName, indexName);
} catch (Exception e) {
@@ -945,22 +963,59 @@ public class CatalogServer extends AbstractService {
}
@Override
- public IndexDescProto getIndexByColumn(RpcController controller, GetIndexByColumnRequest request)
+ public IndexDescProto getIndexByColumnNames(RpcController controller, GetIndexByColumnNamesRequest request)
throws ServiceException {
TableIdentifierProto identifier = request.getTableIdentifier();
String databaseName = identifier.getDatabaseName();
String tableName = identifier.getTableName();
- String columnName = request.getColumnName();
+ List<String> columnNamesList = request.getColumnNamesList();
+ String[] columnNames = new String[columnNamesList.size()];
+ columnNames = columnNamesList.toArray(columnNames);
rlock.lock();
try {
- if (!store.existIndexByColumn(databaseName, tableName, columnName)) {
- throw new NoSuchIndexException(databaseName, columnName);
+ if (!store.existIndexByColumns(databaseName, tableName, columnNames)) {
+ throw new NoSuchIndexException(databaseName, columnNames);
+ }
+ return store.getIndexByColumns(databaseName, tableName, columnNames);
+ } catch (Exception e) {
+ LOG.error("ERROR: cannot get index for " + tableName + "." + columnNames, e);
+ return null;
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ @Override
+ public GetAllIndexesResponse getAllIndexesByTable(RpcController controller, TableIdentifierProto request)
+ throws ServiceException {
+ rlock.lock();
+ String databaseName = request.getDatabaseName();
+ String tableName = request.getTableName();
+ try {
+ GetAllIndexesResponse.Builder builder = GetAllIndexesResponse.newBuilder();
+ for (String eachIndexName : store.getAllIndexNamesByTable(databaseName, tableName)) {
+ builder.addIndexDesc(store.getIndexByName(databaseName, eachIndexName));
}
- return store.getIndexByColumn(databaseName, tableName, columnName);
+ return builder.build();
} catch (Exception e) {
- LOG.error("ERROR : cannot get index for " + tableName + "." + columnName, e);
+ LOG.error("ERROR: cannot get all indexes for " + databaseName + "." + tableName, e);
+ return null;
+ } finally {
+ rlock.unlock();
+ }
+ }
+
+ @Override
+ public GetIndexesProto getAllIndexes(RpcController controller, NullProto request) throws ServiceException {
+ rlock.lock();
+ try {
+ GetIndexesProto.Builder builder = GetIndexesProto.newBuilder();
+ builder.addAllIndex(store.getAllIndexes());
+ return builder.build();
+ } catch (Exception e) {
+ LOG.error("ERROR: cannot get all indexes", e);
return null;
} finally {
rlock.unlock();
@@ -988,18 +1043,6 @@ public class CatalogServer extends AbstractService {
return BOOL_TRUE;
}
-
- @Override
- public GetIndexesProto getAllIndexes(RpcController controller, NullProto request) throws ServiceException {
- rlock.lock();
- try {
- return GetIndexesProto.newBuilder().addAllIndex(store.getAllIndexes()).build();
- } catch (Exception e) {
- throw new ServiceException(e);
- } finally {
- rlock.unlock();
- }
- }
public boolean checkIfBuiltin(FunctionType type) {
return type == GENERAL || type == AGGREGATION || type == DISTINCT_AGGREGATION;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java
index a079a93..d527b19 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java
@@ -27,12 +27,8 @@ class IndexesTableDescriptor extends AbstractTableDescriptor {
new ColumnDescriptor("db_id", Type.INT4, 0),
new ColumnDescriptor("tid", Type.INT4, 0),
new ColumnDescriptor("index_name", Type.TEXT, 0),
- new ColumnDescriptor("column_name", Type.TEXT, 0),
- new ColumnDescriptor("data_type", Type.TEXT, 0),
- new ColumnDescriptor("index_type", Type.TEXT, 0),
- new ColumnDescriptor("is_unique", Type.BOOLEAN, 0),
- new ColumnDescriptor("is_clustered", Type.BOOLEAN, 0),
- new ColumnDescriptor("is_ascending", Type.BOOLEAN, 0)
+ new ColumnDescriptor("index_method", Type.TEXT, 0),
+ new ColumnDescriptor("index_path", Type.TEXT, 0),
};
public IndexesTableDescriptor(InfoSchemaMetadataDictionary metadataDictionary) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index b541e67..0cd8803 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -27,9 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.tajo.annotation.Nullable;
-import org.apache.tajo.catalog.CatalogConstants;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.FunctionDesc;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
@@ -1968,37 +1966,55 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
Connection conn = null;
PreparedStatement pstmt = null;
- String databaseName = proto.getTableIdentifier().getDatabaseName();
- String tableName = proto.getTableIdentifier().getTableName();
- String columnName = CatalogUtil.extractSimpleName(proto.getColumn().getName());
+ final String databaseName = proto.getTableIdentifier().getDatabaseName();
+ final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName());
try {
+ // indexes table
int databaseId = getDatabaseId(databaseName);
int tableId = getTableId(databaseId, databaseName, tableName);
String sql = "INSERT INTO " + TB_INDEXES +
" (" + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " +
- "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) " +
- "VALUES (?,?,?,?,?,?,?,?,?,?)";
+ "INDEX_TYPE, PATH, COLUMN_NAMES, DATA_TYPES, ORDERS, NULL_ORDERS, IS_UNIQUE, IS_CLUSTERED) " +
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
+ StringBuilder columnNamesBuilder = new StringBuilder();
+ StringBuilder dataTypesBuilder= new StringBuilder();
+ StringBuilder ordersBuilder = new StringBuilder();
+ StringBuilder nullOrdersBuilder = new StringBuilder();
+ for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) {
+ // Since the key columns are always sorted in order of their occurrence position in the relation schema,
+ // the concatenated name can be uniquely identified.
+ columnNamesBuilder.append(columnSpec.getColumn().getName()).append(",");
+ dataTypesBuilder.append(columnSpec.getColumn().getDataType().getType().name()).append(",");
+ ordersBuilder.append(columnSpec.getAscending()).append(",");
+ nullOrdersBuilder.append(columnSpec.getNullFirst()).append(",");
+ }
+ columnNamesBuilder.deleteCharAt(columnNamesBuilder.length()-1);
+ dataTypesBuilder.deleteCharAt(dataTypesBuilder.length()-1);
+ ordersBuilder.deleteCharAt(ordersBuilder.length()-1);
+ nullOrdersBuilder.deleteCharAt(nullOrdersBuilder.length()-1);
+
conn = getConnection();
conn.setAutoCommit(false);
pstmt = conn.prepareStatement(sql);
pstmt.setInt(1, databaseId);
pstmt.setInt(2, tableId);
- pstmt.setString(3, proto.getName());
- pstmt.setString(4, columnName);
- pstmt.setString(5, proto.getColumn().getDataType().getType().name());
- pstmt.setString(6, proto.getIndexMethod().toString());
- pstmt.setString(7, proto.getIndexPath());
- pstmt.setBoolean(8, proto.hasIsUnique() && proto.getIsUnique());
- pstmt.setBoolean(9, proto.hasIsClustered() && proto.getIsClustered());
- pstmt.setBoolean(10, proto.hasIsAscending() && proto.getIsAscending());
+ pstmt.setString(3, proto.getIndexName()); // index name
+ pstmt.setString(4, proto.getIndexMethod().toString()); // index type
+ pstmt.setString(5, proto.getIndexPath()); // index path
+ pstmt.setString(6, columnNamesBuilder.toString());
+ pstmt.setString(7, dataTypesBuilder.toString());
+ pstmt.setString(8, ordersBuilder.toString());
+ pstmt.setString(9, nullOrdersBuilder.toString());
+ pstmt.setBoolean(10, proto.hasIsUnique() && proto.getIsUnique());
+ pstmt.setBoolean(11, proto.hasIsClustered() && proto.getIsClustered());
pstmt.executeUpdate();
conn.commit();
} catch (SQLException se) {
@@ -2052,9 +2068,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
final static String GET_INDEXES_SQL =
- "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, " +
- "IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES;
-
+ "SELECT * FROM " + TB_INDEXES;
@Override
public IndexDescProto getIndexByName(String databaseName, final String indexName)
@@ -2085,6 +2099,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
resultToIndexDescProtoBuilder(builder, res);
String tableName = getTableName(conn, res.getInt(COL_TABLES_PK));
builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
+ builder.setTargetRelationSchema(getTable(databaseName, tableName).getSchema());
proto = builder.build();
} catch (SQLException se) {
throw new CatalogException(se);
@@ -2096,9 +2111,8 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public IndexDescProto getIndexByColumn(final String databaseName,
- final String tableName,
- final String columnName) throws CatalogException {
+ public IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames)
+ throws CatalogException {
Connection conn = null;
ResultSet res = null;
PreparedStatement pstmt = null;
@@ -2106,25 +2120,35 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
try {
int databaseId = getDatabaseId(databaseName);
+ int tableId = getTableId(databaseId, databaseName, tableName);
+ TableDescProto tableDescProto = getTable(databaseName, tableName);
- String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?";
+ String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " +
+ COL_TABLES_PK + "=? AND COLUMN_NAMES=?";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
+ // Since the column names in the unified name are always sorted
+ // in order of occurrence position in the relation schema,
+ // they can be uniquely identified.
+ String unifiedName = CatalogUtil.buildFQName(databaseName, tableName,
+ CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames));
conn = getConnection();
pstmt = conn.prepareStatement(sql);
pstmt.setInt(1, databaseId);
- ;
- pstmt.setString(2, columnName);
+ pstmt.setInt(2, tableId);
+ pstmt.setString(3, unifiedName);
res = pstmt.executeQuery();
if (!res.next()) {
- throw new CatalogException("ERROR: there is no index matched to " + columnName);
+ throw new CatalogException("ERROR: there is no index matched to " + unifiedName);
}
+
IndexDescProto.Builder builder = IndexDescProto.newBuilder();
resultToIndexDescProtoBuilder(builder, res);
builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName));
+ builder.setTargetRelationSchema(tableDescProto.getSchema());
proto = builder.build();
} catch (SQLException se) {
throw new CatalogException(se);
@@ -2169,7 +2193,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public boolean existIndexByColumn(String databaseName, String tableName, String columnName)
+ public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames)
throws CatalogException {
Connection conn = null;
ResultSet res = null;
@@ -2179,18 +2203,27 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
try {
int databaseId = getDatabaseId(databaseName);
+ int tableId = getTableId(databaseId, databaseName, tableName);
+ Schema relationSchema = new Schema(getTable(databaseName, tableName).getSchema());
String sql =
- "SELECT INDEX_NAME FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?";
+ "SELECT " + COL_INDEXES_PK + " FROM " + TB_INDEXES +
+ " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=? AND COLUMN_NAMES=?";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
+ // Since the column names in the unified name are always sorted
+ // in order of occurrence position in the relation schema,
+ // they can be uniquely identified.
+ String unifiedName = CatalogUtil.buildFQName(databaseName, tableName,
+ CatalogUtil.getUnifiedSimpleColumnName(new Schema(relationSchema), columnNames));
conn = getConnection();
pstmt = conn.prepareStatement(sql);
pstmt.setInt(1, databaseId);
- pstmt.setString(2, columnName);
+ pstmt.setInt(2, tableId);
+ pstmt.setString(3, unifiedName);
res = pstmt.executeQuery();
exist = res.next();
} catch (SQLException se) {
@@ -2202,22 +2235,18 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
@Override
- public IndexDescProto[] getIndexes(String databaseName, final String tableName)
+ public List<String> getAllIndexNamesByTable(final String databaseName, final String tableName)
throws CatalogException {
- Connection conn = null;
ResultSet res = null;
PreparedStatement pstmt = null;
- final List<IndexDescProto> protos = new ArrayList<IndexDescProto>();
+ final List<String> indexNames = new ArrayList<String>();
try {
final int databaseId = getDatabaseId(databaseName);
final int tableId = getTableId(databaseId, databaseName, tableName);
- final TableIdentifierProto tableIdentifier = CatalogUtil.buildTableIdentifier(databaseName, tableName);
-
String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=?";
-
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
}
@@ -2229,10 +2258,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
res = pstmt.executeQuery();
while (res.next()) {
- IndexDescProto.Builder builder = IndexDescProto.newBuilder();
- resultToIndexDescProtoBuilder(builder, res);
- builder.setTableIdentifier(tableIdentifier);
- protos.add(builder.build());
+ indexNames.add(res.getString("index_name"));
}
} catch (SQLException se) {
throw new CatalogException(se);
@@ -2240,57 +2266,73 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
CatalogUtil.closeQuietly(pstmt, res);
}
- return protos.toArray(new IndexDescProto[protos.size()]);
+ return indexNames;
}
-
- @Override
- public List<IndexProto> getAllIndexes() throws CatalogException {
- Connection conn = null;
- Statement stmt = null;
- ResultSet resultSet = null;
- List<IndexProto> indexes = new ArrayList<IndexProto>();
+ @Override
+ public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException {
+ ResultSet res = null;
+ PreparedStatement pstmt = null;
+ final List<String> indexNames = new ArrayList<String>();
try {
- String sql = "SELECT " + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " +
- "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES;
+ final int databaseId = getDatabaseId(databaseName);
+ final int tableId = getTableId(databaseId, databaseName, tableName);
- conn = getConnection();
- stmt = conn.createStatement();
- resultSet = stmt.executeQuery(sql);
- while (resultSet.next()) {
- IndexProto.Builder builder = IndexProto.newBuilder();
-
- builder.setDbId(resultSet.getInt(COL_DATABASES_PK));
- builder.setTId(resultSet.getInt(COL_TABLES_PK));
- builder.setIndexName(resultSet.getString("INDEX_NAME"));
- builder.setColumnName(resultSet.getString("COLUMN_NAME"));
- builder.setDataType(resultSet.getString("DATA_TYPE"));
- builder.setIndexType(resultSet.getString("INDEX_TYPE"));
- builder.setIsUnique(resultSet.getBoolean("IS_UNIQUE"));
- builder.setIsClustered(resultSet.getBoolean("IS_CLUSTERED"));
- builder.setIsAscending(resultSet.getBoolean("IS_ASCENDING"));
-
- indexes.add(builder.build());
+ String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=?";
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(sql);
}
+
+ conn = getConnection();
+ pstmt = conn.prepareStatement(sql);
+ pstmt.setInt(1, databaseId);
+ pstmt.setInt(2, tableId);
+ res = pstmt.executeQuery();
+
+ return res.next();
} catch (SQLException se) {
throw new CatalogException(se);
} finally {
- CatalogUtil.closeQuietly(stmt, resultSet);
+ CatalogUtil.closeQuietly(pstmt, res);
}
-
- return indexes;
+ }
+
+ @Override
+ public List<IndexDescProto> getAllIndexes() throws CatalogException {
+ List<IndexDescProto> indexDescProtos = TUtil.newList();
+ for (String databaseName : getAllDatabaseNames()) {
+ for (String tableName : getAllTableNames(databaseName)) {
+ for (String indexName: getAllIndexNamesByTable(databaseName, tableName)) {
+ indexDescProtos.add(getIndexByName(databaseName, indexName));
+ }
+ }
+ }
+ return indexDescProtos;
}
private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder,
final ResultSet res) throws SQLException {
- builder.setName(res.getString("index_name"));
- builder.setColumn(indexResultToColumnProto(res));
+ builder.setIndexName(res.getString("index_name"));
builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim()));
builder.setIndexPath(res.getString("path"));
+ String[] columnNames, dataTypes, orders, nullOrders;
+ columnNames = res.getString("column_names").trim().split(",");
+ dataTypes = res.getString("data_types").trim().split(",");
+ orders = res.getString("orders").trim().split(",");
+ nullOrders = res.getString("null_orders").trim().split(",");
+ int columnNum = columnNames.length;
+ for (int i = 0; i < columnNum; i++) {
+ SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder();
+ colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(columnNames[i])
+ .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build());
+ colSpecBuilder.setAscending(orders[i].equals("true"));
+ colSpecBuilder.setNullFirst(nullOrders[i].equals("true"));
+ builder.addKeySortSpecs(colSpecBuilder.build());
+ }
builder.setIsUnique(res.getBoolean("is_unique"));
builder.setIsClustered(res.getBoolean("is_clustered"));
- builder.setIsAscending(res.getBoolean("is_ascending"));
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
index ed6fedc..ed1cbc4 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java
@@ -20,14 +20,7 @@ package org.apache.tajo.catalog.store;
import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import java.io.Closeable;
@@ -36,10 +29,6 @@ import org.apache.tajo.catalog.exception.CatalogException;
import java.util.Collection;
import java.util.List;
-import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
-import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionMethodProto;
-import static org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto;
-
public interface CatalogStore extends Closeable {
/*************************** Tablespace ******************************/
void createTablespace(String spaceName, String spaceUri) throws CatalogException;
@@ -129,18 +118,20 @@ public interface CatalogStore extends Closeable {
void dropIndex(String databaseName, String indexName) throws CatalogException;
IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException;
-
- IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName)
+
+ IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames)
throws CatalogException;
boolean existIndexByName(String databaseName, String indexName) throws CatalogException;
-
- boolean existIndexByColumn(String databaseName, String tableName, String columnName)
+
+ boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames)
throws CatalogException;
- IndexDescProto [] getIndexes(String databaseName, String tableName) throws CatalogException;
-
- List<IndexProto> getAllIndexes() throws CatalogException;
+ List<String> getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException;
+
+ boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException;
+
+ List<IndexDescProto> getAllIndexes() throws CatalogException;
/************************** FUNCTION *****************************/
[2/5] tajo git commit: TAJO-838: Improve query planner to utilize
index. (jihoon)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
index 2a66909..83e33b7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.tajo.IntegrationTest;
import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.IndexDesc;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -38,35 +39,60 @@ public class TestCreateIndex extends QueryTestCaseBase {
super(TajoConstants.DEFAULT_DATABASE_NAME);
}
- private static void assertIndexExist(String indexName) throws IOException {
- Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName);
+ private static void assertIndexNotExist(String databaseName, String indexName) throws IOException {
+ Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), databaseName + "/" + indexName);
FileSystem fs = indexPath.getFileSystem(conf);
- assertTrue(fs.exists(indexPath));
- assertEquals(2, fs.listStatus(indexPath).length);
- fs.deleteOnExit(indexPath);
+ if (fs.exists(indexPath)) {
+ fs.deleteOnExit(indexPath);
+ assertFalse("Index is not deleted from the file system.", true);
+ }
}
@Test
public final void testCreateIndex() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_idx");
+ assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx"));
+ assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"}));
+ executeString("drop index l_orderkey_idx");
+ assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx"));
+ assertIndexNotExist(getCurrentDatabase(), "l_orderkey_idx");
}
@Test
public final void testCreateIndexOnMultiAttrs() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_partkey_idx");
+ assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_idx"));
+ assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey", "l_partkey"}));
+ executeString("drop index l_orderkey_partkey_idx");
+ assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_idx"));
+ assertIndexNotExist(getCurrentDatabase(), "l_orderkey_partkey_idx");
}
@Test
public final void testCreateIndexWithCondition() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_partkey_lt10_idx");
+ assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx"));
+ assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey", "l_partkey"}));
+ executeString("drop index l_orderkey_partkey_lt10_idx");
+ assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx"));
+ assertIndexNotExist(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx");
}
@Test
public final void testCreateIndexOnExpression() throws Exception {
executeQuery();
- assertIndexExist("l_orderkey_100_lt10_idx");
+ assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_lt10_idx"));
+ executeString("drop index l_orderkey_100_lt10_idx");
+ assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_lt10_idx"));
+ assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_lt10_idx");
+ }
+
+ @Test
+ public final void testCreateIndexOnMultiExprs() throws Exception {
+ executeQuery();
+ assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx"));
+ executeString("drop index l_orderkey_100_l_linenumber_10_lt10_idx");
+ assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx"));
+ assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java
new file mode 100644
index 0000000..f94a3b5
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import com.google.protobuf.ServiceException;
+import org.apache.tajo.IntegrationTest;
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.SessionVars;
+import org.apache.tajo.TajoConstants;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+
+@Category(IntegrationTest.class)
+public class TestIndexScan extends QueryTestCaseBase {
+
+ public TestIndexScan() throws ServiceException {
+ super(TajoConstants.DEFAULT_DATABASE_NAME);
+ Map<String,String> sessionVars = new HashMap<String, String>();
+ sessionVars.put(SessionVars.INDEX_ENABLED.keyname(), "true");
+ sessionVars.put(SessionVars.INDEX_SELECTIVITY_THRESHOLD.keyname(), "0.01f");
+ client.updateSessionVariables(sessionVars);
+ }
+
+ @Test
+ public final void testOnSortedNonUniqueKeys() throws Exception {
+ executeString("create index l_orderkey_idx on lineitem (l_orderkey)");
+ ResultSet res = executeString("select * from lineitem where l_orderkey = 1;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_orderkey_idx");
+ }
+
+ @Test
+ public final void testOnUnsortedTextKeys() throws Exception {
+ executeString("create index l_shipdate_idx on lineitem (l_shipdate)");
+ ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem where l_shipdate = '1997-01-28';");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_shipdate_idx");
+ }
+
+ @Test
+ public final void testOnMultipleKeys() throws Exception {
+ executeString("create index multikey_idx on lineitem (l_shipdate asc null last, l_tax desc null first, l_shipmode, l_linenumber desc null last)");
+ ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem " +
+ "where l_shipdate = '1997-01-28' and l_tax = 0.05 and l_shipmode = 'RAIL' and l_linenumber = 1;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index multikey_idx");
+ }
+
+ @Test
+ public final void testOnMultipleKeys2() throws Exception {
+ executeString("create index multikey_idx on lineitem (l_shipdate asc null last, l_tax desc null first)");
+ ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem " +
+ "where l_shipdate = '1997-01-28' and l_tax = 0.05 and l_shipmode = 'RAIL' and l_linenumber = 1;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index multikey_idx");
+ }
+
+ @Test
+ public final void testOnMultipleExprs() throws Exception {
+ executeString("create index l_orderkey_100_l_linenumber_10_idx on lineitem (l_orderkey*100-l_linenumber*10 asc null first);");
+ ResultSet res = executeString("select l_orderkey, l_linenumber from lineitem where l_orderkey*100-l_linenumber*10 = 280");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_orderkey_100_l_linenumber_10_idx");
+ }
+
+ @Test
+ public final void testWithGroupBy() throws Exception {
+ executeString("create index l_shipdate_idx on lineitem (l_shipdate)");
+ ResultSet res = executeString("select l_shipdate, count(*) from lineitem where l_shipdate = '1997-01-28' group by l_shipdate;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_shipdate_idx");
+ }
+
+ @Test
+ public final void testWithSort() throws Exception {
+ executeString("create index l_orderkey_idx on lineitem (l_orderkey)");
+ ResultSet res = executeString("select l_shipdate from lineitem where l_orderkey = 1 order by l_shipdate;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_orderkey_idx");
+ }
+
+ @Test
+ public final void testWithJoin() throws Exception {
+ executeString("create index l_orderkey_idx on lineitem (l_orderkey)");
+ executeString("create index o_orderkey_idx on orders (o_orderkey)");
+ ResultSet res = executeString("select l_shipdate, o_orderstatus from lineitem, orders where l_orderkey = o_orderkey and l_orderkey = 1 and o_orderkey = 1;");
+ assertResultSet(res);
+ cleanupQuery(res);
+ executeString("drop index l_orderkey_idx");
+ executeString("drop index o_orderkey_idx");
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 3400752..0bfcb09 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -766,8 +766,8 @@ public class TestTablePartitions extends QueryTestCaseBase {
ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName
+ " select l_orderkey, l_partkey from lineitem");
- assertTrue(response.hasErrorMessage());
- assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n");
+ assertTrue(response.getResult().hasErrorMessage());
+ assertEquals(response.getResult().getErrorMessage(), "INSERT has smaller expressions than target columns\n");
res = executeFile("case14.sql");
assertResultSet(res, "case14.result");
@@ -786,8 +786,8 @@ public class TestTablePartitions extends QueryTestCaseBase {
ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName
+ " select l_returnflag , l_orderkey, l_partkey from lineitem");
- assertTrue(response.hasErrorMessage());
- assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n");
+ assertTrue(response.getResult().hasErrorMessage());
+ assertEquals(response.getResult().getErrorMessage(), "INSERT has smaller expressions than target columns\n");
res = executeFile("case15.sql");
assertResultSet(res, "case15.result");
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
index 712243b..760cb4c 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
@@ -80,7 +80,7 @@ public class TestExecutionBlockCursor {
analyzer = new SQLAnalyzer();
logicalPlanner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
StorageManager sm = StorageManager.getFileStorageManager(conf);
dispatcher = new AsyncDispatcher();
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
index d0f7cf4..e2f3417 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java
@@ -115,7 +115,7 @@ public class TestGlobalPlanner {
sqlAnalyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(util.getConfiguration());
+ optimizer = new LogicalOptimizer(util.getConfiguration(), catalog);
globalPlanner = new GlobalPlanner(util.getConfiguration(), catalog);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java
index 8ca4cff..e728207 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java
@@ -76,7 +76,7 @@ public class TestKillQuery {
String query = "select l_orderkey, l_partkey from lineitem group by l_orderkey, l_partkey order by l_orderkey";
LogicalPlanner planner = new LogicalPlanner(catalog);
- LogicalOptimizer optimizer = new LogicalOptimizer(conf);
+ LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog);
Expr expr = analyzer.parse(query);
LogicalPlan plan = planner.createPlan(defaultContext, expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index 200ba31..2eecd4d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -94,7 +94,7 @@ public class TestRangeRetrieverHandler {
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
- optimizer = new LogicalOptimizer(conf);
+ optimizer = new LogicalOptimizer(conf, catalog);
schema = new Schema();
schema.addColumn("empid", Type.INT4);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql
new file mode 100644
index 0000000..7938005
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql
@@ -0,0 +1 @@
+create index l_orderkey_100_l_linenumber_10_lt10_idx on lineitem (l_orderkey*100-l_linenumber*10 asc null first) where l_orderkey*100 > 10;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result
new file mode 100644
index 0000000..bcb645d
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result
@@ -0,0 +1,3 @@
+l_orderkey,l_linenumber
+-------------------------------
+3,2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result
new file mode 100644
index 0000000..86d468e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result
@@ -0,0 +1,3 @@
+l_orderkey,l_shipdate,l_comment
+-------------------------------
+2,1997-01-28,ven requests. deposits breach a
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result
new file mode 100644
index 0000000..86d468e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result
@@ -0,0 +1,3 @@
+l_orderkey,l_shipdate,l_comment
+-------------------------------
+2,1997-01-28,ven requests. deposits breach a
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result
new file mode 100644
index 0000000..fb8a4c2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result
@@ -0,0 +1,4 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result
new file mode 100644
index 0000000..86d468e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result
@@ -0,0 +1,3 @@
+l_orderkey,l_shipdate,l_comment
+-------------------------------
+2,1997-01-28,ven requests. deposits breach a
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result
new file mode 100644
index 0000000..2d1d12e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result
@@ -0,0 +1,3 @@
+l_shipdate,?count
+-------------------------------
+1997-01-28,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result
new file mode 100644
index 0000000..d119969
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result
@@ -0,0 +1,4 @@
+l_shipdate,o_orderstatus
+-------------------------------
+1996-03-13,O
+1996-04-12,O
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result
new file mode 100644
index 0000000..774a411
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result
@@ -0,0 +1,4 @@
+l_shipdate
+-------------------------------
+1996-03-13
+1996-04-12
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
index b5b7c22..90f948e 100644
--- a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
@@ -34,6 +34,8 @@ Available Session Variables:
\set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite.
\set NULL_CHAR [text value] - null char of text file output
\set CODEGEN [true or false] - Runtime code generation enabled (experiment)
+\set INDEX_ENABLED [true or false] - index scan enabled
+\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan
\set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs.
\set FETCH_ROWNUM [int value] - Sets the number of rows at a time from Master
\set DEBUG_ENABLED [true or false] - (debug only) debug mode enabled
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/index.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst
index 80cd842..667f270 100644
--- a/tajo-docs/src/main/sphinx/index.rst
+++ b/tajo-docs/src/main/sphinx/index.rst
@@ -37,6 +37,7 @@ Table of Contents:
functions
table_management
table_partitioning
+ index_overview
backup_and_restore
hcatalog_integration
jdbc_driver
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/index/future_work.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/future_work.rst b/tajo-docs/src/main/sphinx/index/future_work.rst
new file mode 100644
index 0000000..c6ec47d
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/future_work.rst
@@ -0,0 +1,8 @@
+*************************************
+Future Works
+*************************************
+
+* Providing more index types, such as bitmap and HBase index
+* Supporting index on partitioned tables
+* Supporting the backup and restore feature
+* Cost-based query optimization by estimating the query selectivity
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/index/how_to_use.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/how_to_use.rst b/tajo-docs/src/main/sphinx/index/how_to_use.rst
new file mode 100644
index 0000000..82a8bc9
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/how_to_use.rst
@@ -0,0 +1,69 @@
+*************************************
+How to use index?
+*************************************
+
+-------------------------------------
+1. Create index
+-------------------------------------
+
+The first step for utilizing index is index creation. You can create index using SQL (:doc:`/sql_language/ddl`) or Tajo API (:doc:`/tajo_client_api`). For example, you can create a BST index on the lineitem table by submitting the following SQL to Tajo.
+
+.. code-block:: sql
+
+ create index l_orderkey_idx on lineitem (l_orderkey);
+
+If the index is created successfully, you can see the information about that index as follows: ::
+
+ default> \d lineitem
+
+ table name: default.lineitem
+ table path: hdfs://localhost:7020/tpch/lineitem
+ store type: CSV
+ number of rows: unknown
+ volume: 753.9 MB
+ Options:
+ 'text.delimiter'='|'
+
+ schema:
+ l_orderkey INT8
+ l_partkey INT8
+ l_suppkey INT8
+ l_linenumber INT8
+ l_quantity FLOAT4
+ l_extendedprice FLOAT4
+ l_discount FLOAT4
+ l_tax FLOAT4
+ l_returnflag TEXT
+ l_linestatus TEXT
+ l_shipdate DATE
+ l_commitdate DATE
+ l_receiptdate DATE
+ l_shipinstruct TEXT
+ l_shipmode TEXT
+ l_comment TEXT
+
+
+ Indexes:
+ "l_orderkey_idx" TWO_LEVEL_BIN_TREE (l_orderkey ASC NULLS LAST )
+
+For more information about index creation, please refer to the above links.
+
+-------------------------------------
+2. Enable/disable index scans
+-------------------------------------
+
+When an index is successfully created, you must enable the index scan feature as follows:
+
+.. code-block:: sql
+
+ \set INDEX_ENABLED true
+
+If you don't want to use the index scan feature anymore, you can simply disable it as follows:
+
+.. code-block:: sql
+
+ \set INDEX_ENABLED false
+
+.. note::
+
+ If the index scan feature is enabled, Tajo currently always performs index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/index/types.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/types.rst b/tajo-docs/src/main/sphinx/index/types.rst
new file mode 100644
index 0000000..457f453
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/types.rst
@@ -0,0 +1,7 @@
+*************************************
+Index Types
+*************************************
+
+Currently, Tajo supports only one type of index, ``TWO_LEVEL_BIN_TREE``, shortly ``BST``. The BST index is a kind of binary search tree which is extended to be permanently stored on disk. It consists of two levels of nodes; a leaf node indexes the keys with the positions of data in an HDFS block and a root node indexes the keys with the leaf node indices.
+
+When an index scan is started, the query engine first reads the root node and finds the search key. If it finds a leaf node corresponding to the search key, it subsequently finds the search key in that leaf node. Finally, it directly reads a tuple corresponding to the search key from HDFS.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/index_overview.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index_overview.rst b/tajo-docs/src/main/sphinx/index_overview.rst
new file mode 100644
index 0000000..78ecb49
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index_overview.rst
@@ -0,0 +1,20 @@
+***********************
+Index (Experimental Feature)
+***********************
+
+An index is a data structure that is used for efficient query processing. Using an index, the Tajo query engine can directly retrieve search values.
+
+This is still an experimental feature. In order to use indexes, you must check out the source code of the ``index_support`` branch::
+
+ git clone -b index_support https://git-wip-us.apache.org/repos/asf/tajo.git tajo-index
+
+For the source code build, please refer to :doc:`getting_started`.
+
+The following sections describe the supported index types, the query execution with an index, and the future works.
+
+.. toctree::
+ :maxdepth: 1
+
+ index/types
+ index/how_to_use
+ index/future_work
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-docs/src/main/sphinx/sql_language/ddl.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/sql_language/ddl.rst b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
index 3fba6be..60b7190 100644
--- a/tajo-docs/src/main/sphinx/sql_language/ddl.rst
+++ b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
@@ -75,4 +75,35 @@ If you want to add an external table that contains compressed data, you should g
DROP TABLE [IF EXISTS] <table_name> [PURGE]
-``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents.
\ No newline at end of file
+``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents.
+
+========================
+ CREATE INDEX
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ CREATE INDEX [ name ] ON table_name [ USING method ]
+ ( { column_name | ( expression ) } [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [, ...] )
+ [ WHERE predicate ]
+
+------------------------
+ Index method
+------------------------
+
+Currently, Tajo supports only one type of index.
+
+Index methods:
+ * TWO_LEVEL_BIN_TREE: This method is used by default in Tajo. For more information about its structure, please refer to :doc:`/index/types`.
+
+========================
+ DROP INDEX
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ DROP INDEX name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index 18a8859..ed16f82 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -28,6 +28,7 @@ import org.apache.tajo.ConfigKey;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.JoinType;
+import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.util.ReflectionUtil;
@@ -39,6 +40,10 @@ import org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm;
import org.apache.tajo.plan.joinorder.JoinGraph;
import org.apache.tajo.plan.joinorder.JoinOrderAlgorithm;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.rules.AccessPathRewriter;
+import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule;
+import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter;
+import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.rewrite.*;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
@@ -57,12 +62,15 @@ import static org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm.g
public class LogicalOptimizer {
private static final Log LOG = LogFactory.getLog(LogicalOptimizer.class.getName());
+ private CatalogService catalog;
private BaseLogicalPlanRewriteEngine rulesBeforeJoinOpt;
private BaseLogicalPlanRewriteEngine rulesAfterToJoinOpt;
private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm();
- public LogicalOptimizer(TajoConf conf) {
+ public LogicalOptimizer(TajoConf conf, CatalogService catalog) {
+ this.catalog = catalog;
+ // TODO: set the catalog instance to FilterPushdownRule
Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS);
LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz, conf);
@@ -86,7 +94,7 @@ public class LogicalOptimizer {
}
public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException {
- rulesBeforeJoinOpt.rewrite(context, plan);
+ rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog));
DirectedGraphCursor<String, BlockEdge> blockCursor =
new DirectedGraphCursor<String, BlockEdge>(plan.getQueryBlockGraph(), plan.getRootBlock().getName());
@@ -99,7 +107,7 @@ public class LogicalOptimizer {
} else {
LOG.info("Skip Join Optimized.");
}
- rulesAfterToJoinOpt.rewrite(context, plan);
+ rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog));
return plan.getRootBlock().getRoot();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
index 3baf61d..1535882 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java
@@ -25,6 +25,8 @@ import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.NotThreadSafe;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.plan.expr.AlgebraicUtil.IdentifiableNameBuilder;
+import org.apache.tajo.plan.rewrite.rules.AccessPathInfo;
import org.apache.tajo.util.graph.DirectedGraphCursor;
import org.apache.tajo.util.graph.SimpleDirectedGraph;
import org.apache.tajo.plan.expr.ConstEval;
@@ -150,6 +152,8 @@ public class LogicalPlan {
/**
* It generates an unique column name from Expr. It is usually used for an expression or predicate without
* a specified name (i.e., alias).
+ * Here, some expressions require to be identified with their names in the future.
+ * For example, expressions must be identifiable with their names when getting targets in {@link LogicalPlanner#visitCreateIndex}.
*/
public String generateUniqueColumnName(Expr expr) {
String generatedName;
@@ -161,6 +165,11 @@ public class LogicalPlan {
return generatedName;
}
+ private String generateUniqueIdentifiableColumnName(Expr expr) {
+ IdentifiableNameBuilder nameBuilder = new IdentifiableNameBuilder(expr);
+ return nameBuilder.build();
+ }
+
/**
* It attaches a generated column name with a sequence id. It always keeps generated names unique.
*/
@@ -408,6 +417,7 @@ public class LogicalPlan {
private final Map<String, String> columnAliasMap = TUtil.newHashMap();
private final Map<OpType, List<Expr>> operatorToExprMap = TUtil.newHashMap();
private final List<RelationNode> relationList = TUtil.newList();
+ private final Map<Integer, List<AccessPathInfo>> relNodePidAccessPathMap = TUtil.newHashMap();
private boolean hasWindowFunction = false;
private final Map<String, ConstEval> constantPoolByRef = Maps.newHashMap();
private final Map<Expr, String> constantPool = Maps.newHashMap();
@@ -496,12 +506,30 @@ public class LogicalPlan {
}
canonicalNameToRelationMap.put(relation.getCanonicalName(), relation);
relationList.add(relation);
+ relNodePidAccessPathMap.put(relation.getPID(), new ArrayList<AccessPathInfo>());
+ }
+
+ public void addRelation(RelationNode relation, List<AccessPathInfo> accessPathInfos) {
+ if (relation.hasAlias()) {
+ TUtil.putToNestedList(relationAliasMap, relation.getTableName(), relation.getCanonicalName());
+ }
+ canonicalNameToRelationMap.put(relation.getCanonicalName(), relation);
+ relationList.add(relation);
+ relNodePidAccessPathMap.put(relation.getPID(), new ArrayList<AccessPathInfo>());
+ }
+
+ public void addAccessPath(RelationNode relation, AccessPathInfo accessPathInfo) {
+ relNodePidAccessPathMap.get(relation.getPID()).add(accessPathInfo);
}
public Collection<RelationNode> getRelations() {
return Collections.unmodifiableList(relationList);
}
+ public List<AccessPathInfo> getAccessInfos(RelationNode relation) {
+ return Collections.unmodifiableList(relNodePidAccessPathMap.get(relation.getPID()));
+ }
+
public boolean hasTableExpression() {
return this.canonicalNameToRelationMap.size() > 0;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
index 6e2b59a..76a32a2 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
@@ -231,7 +231,8 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
return projectionNode;
}
- private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs) throws PlanningException {
+ private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs)
+ throws PlanningException {
Target [] targets = new Target[exprs.length];
for (int i = 0; i < exprs.length; i++) {
NamedExpr namedExpr = exprs[i];
@@ -480,6 +481,11 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
return createIndex;
}
+ @Override
+ public LogicalNode visitDropIndex(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, DropIndex expr) {
+ return ctx.plan.createNode(DropIndexNode.class);
+ }
+
public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, TruncateTable expr)
throws PlanningException {
TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 3604e06..b50ef78 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -41,7 +41,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.plan.LogicalPlan.QueryBlock;
import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
@@ -52,12 +51,12 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode;
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.TUtil;
+import java.net.URI;
import java.util.*;
import static org.apache.tajo.algebra.CreateTable.PartitionType;
@@ -1930,9 +1929,9 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return alterTableNode;
}
- private static Path getIndexPath(PlanContext context, String databaseName, String indexName) {
+ private static URI getIndexPath(PlanContext context, String databaseName, String indexName) {
return new Path(TajoConf.getWarehouseDir(context.queryContext.getConf()),
- databaseName + "/" + indexName + "/");
+ databaseName + "/" + indexName + "/").toUri();
}
@Override
@@ -1960,15 +1959,20 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey());
}
for (int i = 0; i < sortKeyNum; i++) {
+ // even if base expressions don't have their name,
+ // reference names should be identifiable for the later sort spec creation.
referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr);
block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs);
block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs);
}
- createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs));
- createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase()));
- createIndexNode.setIndexPath(getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE),
- createIndex.getIndexName()));
+ Collection<RelationNode> relations = block.getRelations();
+ assert relations.size() == 1;
+ createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(),
+ annotateSortSpecs(block, referNames, sortSpecs));
+ createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase()));
+ createIndexNode.setIndexPath(
+ getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName()));
if (createIndex.getParams() != null) {
KeyValueSet keyValueSet = new KeyValueSet();
@@ -1981,6 +1985,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
@Override
+ public LogicalNode visitDropIndex(PlanContext context, Stack<Expr> stack, DropIndex dropIndex) {
+ DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex);
+ dropIndexNode.setIndexName(dropIndex.getIndexName());
+ return dropIndexNode;
+ }
+
+ @Override
public LogicalNode visitTruncateTable(PlanContext context, Stack<Expr> stack, TruncateTable truncateTable)
throws PlanningException {
TruncateTableNode truncateTableNode = context.queryBlock.getNodeFromExpr(truncateTable);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java
index 991860f..63c10ae 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java
@@ -141,6 +141,7 @@ public class NamedExprsManager {
/**
* Adds an expression and returns a reference name.
+ * @param expr added expression
*/
public String addExpr(Expr expr) throws PlanningException {
if (idToExprBiMap.inverse().containsKey(expr)) {
@@ -205,7 +206,8 @@ public class NamedExprsManager {
* Adds a list of expressions and returns a list of reference names.
* If some NamedExpr has an alias, NamedExprsManager specifies the alias for the NamedExpr.
*/
- public String [] addNamedExprArray(@Nullable Collection<NamedExpr> namedExprs) throws PlanningException {
+ public String [] addNamedExprArray(@Nullable Collection<NamedExpr> namedExprs)
+ throws PlanningException {
if (namedExprs != null && namedExprs.size() > 0) {
String [] names = new String[namedExprs.size()];
int i = 0;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java
index 6e7a514..d818249 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java
@@ -53,6 +53,7 @@ public interface AlgebraVisitor<CONTEXT, RESULT> {
RESULT visitAlterTablespace(CONTEXT ctx, Stack<Expr> stack, AlterTablespace expr) throws PlanningException;
RESULT visitAlterTable(CONTEXT ctx, Stack<Expr> stack, AlterTable expr) throws PlanningException;
RESULT visitCreateIndex(CONTEXT ctx, Stack<Expr> stack, CreateIndex expr) throws PlanningException;
+ RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) throws PlanningException;
RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException;
// Insert or Update
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
index ffaf713..664dd80 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
@@ -128,6 +128,9 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
case TruncateTable:
current = visitTruncateTable(ctx, stack, (TruncateTable)expr);
break;
+ case DropIndex:
+ current = visitDropIndex(ctx, stack, (DropIndex) expr);
+ break;
case Insert:
current = visitInsert(ctx, stack, (Insert) expr);
@@ -488,6 +491,11 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
}
@Override
+ public RESULT visitDropIndex(CONTEXT ctx, Stack<Expr> stack, DropIndex expr) {
+ return null;
+ }
+
+ @Override
public RESULT visitTruncateTable(CONTEXT ctx, Stack<Expr> stack, TruncateTable expr) throws PlanningException {
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index 84352f0..9e6d78f 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@ -18,7 +18,10 @@
package org.apache.tajo.plan.expr;
+import org.apache.tajo.algebra.*;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.plan.PlanningException;
+import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
import java.util.ArrayList;
import java.util.List;
@@ -414,4 +417,70 @@ public class AlgebraicUtil {
found.add(node);
}
}
+
+ public static class IdentifiableNameBuilder extends SimpleAlgebraVisitor<Object, Object> {
+ private Expr expr;
+ private StringBuilder nameBuilder = new StringBuilder();
+
+ public IdentifiableNameBuilder(Expr expr) {
+ this.expr = expr;
+ }
+
+ public String build() {
+ Stack<Expr> stack = new Stack<Expr>();
+ stack.push(expr);
+ try {
+ this.visit(null, stack, expr);
+ } catch (PlanningException e) {
+
+ }
+ return nameBuilder.deleteCharAt(nameBuilder.length()-1).toString();
+ }
+
+ @Override
+ public Object visitBinaryOperator(Object ctx, Stack<Expr> stack, BinaryOperator expr) throws PlanningException {
+ addIntermExpr(expr);
+ return super.visitBinaryOperator(ctx, stack, expr);
+ }
+
+ private void append(String str) {
+ nameBuilder.append(str).append("_");
+ }
+
+ private void addIntermExpr(Expr expr) {
+ this.append(expr.getType().name());
+ }
+
+ @Override
+ public Object visitColumnReference(Object ctx, Stack<Expr> stack, ColumnReferenceExpr expr)
+ throws PlanningException {
+ this.append(expr.getName());
+ return super.visitColumnReference(ctx, stack, expr);
+ }
+
+ @Override
+ public Object visitLiteral(Object ctx, Stack<Expr> stack, LiteralValue expr) throws PlanningException {
+ this.append(expr.getValue());
+ return super.visitLiteral(ctx, stack, expr);
+ }
+
+ @Override
+ public Object visitNullLiteral(Object ctx, Stack<Expr> stack, NullLiteral expr) throws PlanningException {
+ this.append("null");
+ return super.visitNullLiteral(ctx, stack, expr);
+ }
+
+ @Override
+ public Object visitTimestampLiteral(Object ctx, Stack<Expr> stack, TimestampLiteral expr) throws PlanningException {
+ this.append(expr.getDate().toString());
+ this.append(expr.getTime().toString());
+ return super.visitTimestampLiteral(ctx, stack, expr);
+ }
+
+ @Override
+ public Object visitTimeLiteral(Object ctx, Stack<Expr> stack, TimeLiteral expr) throws PlanningException {
+ this.append(expr.getTime().toString());
+ return super.visitTimeLiteral(ctx, stack, expr);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
index f5c2cbd..5b6ebc7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
@@ -382,7 +382,7 @@ public class EvalTreeUtil {
return !leftQualifier.equals(rightQualifier);
}
- static boolean isSingleColumn(EvalNode evalNode) {
+ public static boolean isSingleColumn(EvalNode evalNode) {
return EvalTreeUtil.findUniqueColumns(evalNode).size() == 1;
}
@@ -517,6 +517,10 @@ public class EvalTreeUtil {
return evalNode;
}
+
+ public List<EvalNode> getEvalNodes() {
+ return evalNodes;
+ }
}
public static class OuterJoinSensitiveEvalFinder extends BasicEvalNodeVisitor<Object, Object> {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java
index a8d364f..fcedf48 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java
@@ -18,91 +18,99 @@
package org.apache.tajo.plan.logical;
-import com.google.common.base.Objects;
import com.google.gson.annotations.Expose;
-import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.IndexMeta;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.util.KeyValueSet;
-import org.apache.tajo.util.TUtil;
+
+import java.net.URI;
import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
public class CreateIndexNode extends UnaryNode implements Cloneable {
- @Expose private boolean isUnique;
- @Expose private String indexName;
- @Expose private Path indexPath;
- @Expose private SortSpec[] sortSpecs;
- @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE;
- @Expose private KeyValueSet options;
+ @Expose private IndexMeta indexMeta;
public CreateIndexNode(int pid) {
super(pid, NodeType.CREATE_INDEX);
+ this.indexMeta = new IndexMeta();
}
public void setUnique(boolean unique) {
- this.isUnique = unique;
+ indexMeta.setUnique(unique);
}
public boolean isUnique() {
- return isUnique;
+ return indexMeta.isUnique();
}
public void setIndexName(String indexName) {
- this.indexName = indexName;
+ indexMeta.setIndexName(indexName);
}
public String getIndexName() {
- return this.indexName;
+ return indexMeta.getIndexName();
}
- public void setIndexPath(Path indexPath) {
- this.indexPath = indexPath;
+ public void setIndexPath(URI indexPath) {
+ indexMeta.setIndexPath(indexPath);
}
- public Path getIndexPath() {
- return this.indexPath;
+ public URI getIndexPath() {
+ return indexMeta.getIndexPath();
}
- public void setSortSpecs(SortSpec[] sortSpecs) {
- this.sortSpecs = sortSpecs;
+ public void setKeySortSpecs(Schema targetRelationSchema, SortSpec[] sortSpecs) {
+ indexMeta.setKeySortSpecs(targetRelationSchema, sortSpecs);
}
- public SortSpec[] getSortSpecs() {
- return this.sortSpecs;
+ public SortSpec[] getKeySortSpecs() {
+ return indexMeta.getKeySortSpecs();
}
- public void setIndexType(IndexMethod indexType) {
- this.indexType = indexType;
+ public void setIndexMethod(IndexMethod indexType) {
+ indexMeta.setIndexMethod(indexType);
}
- public IndexMethod getIndexType() {
- return this.indexType;
+ public IndexMethod getIndexMethod() {
+ return indexMeta.getIndexMethod();
}
public void setOptions(KeyValueSet options) {
- this.options = options;
+ indexMeta.setOptions(options);
}
public KeyValueSet getOptions() {
- return this.options;
+ return indexMeta.getOptions();
+ }
+
+ public Schema getTargetRelationSchema() {
+ return indexMeta.getTargetRelationSchema();
+ }
+
+ public boolean hasOptions() {
+ return indexMeta.getOptions() != null;
+ }
+
+ public void setClustered(boolean clustered) {
+ indexMeta.setClustered(clustered);
+ }
+
+ public boolean isClustered() {
+ return indexMeta.isClustered();
}
@Override
public int hashCode() {
- return Objects.hashCode(isUnique, indexName, indexPath, sortSpecs, indexType, options);
+ return indexMeta.hashCode();
}
@Override
public boolean equals(Object obj) {
if (obj instanceof CreateIndexNode) {
CreateIndexNode other = (CreateIndexNode) obj;
- return this.isUnique == other.isUnique &&
- TUtil.checkEquals(this.indexName, other.indexName) &&
- TUtil.checkEquals(this.indexPath, other.indexPath) &&
- TUtil.checkEquals(this.sortSpecs, other.sortSpecs) &&
- this.indexType.equals(other.indexType) &&
- TUtil.checkEquals(this.options, other.options);
+ return this.indexMeta.equals(other.indexMeta);
}
return false;
}
@@ -110,17 +118,13 @@ public class CreateIndexNode extends UnaryNode implements Cloneable {
@Override
public Object clone() throws CloneNotSupportedException {
CreateIndexNode createIndexNode = (CreateIndexNode) super.clone();
- createIndexNode.isUnique = isUnique;
- createIndexNode.indexName = indexName;
- createIndexNode.indexPath = indexPath;
- createIndexNode.sortSpecs = sortSpecs.clone();
- createIndexNode.indexType = indexType;
- createIndexNode.options = (KeyValueSet) (options != null ? options.clone() : null);
+ createIndexNode.indexMeta = (IndexMeta) this.indexMeta.clone();
return createIndexNode;
}
private String getSortSpecString() {
StringBuilder sb = new StringBuilder("Column [key= ");
+ SortSpec[] sortSpecs = indexMeta.getKeySortSpecs();
for (int i = 0; i < sortSpecs.length; i++) {
sb.append(sortSpecs[i].getSortKey().getQualifiedName()).append(" ")
.append(sortSpecs[i].isAscending() ? "asc" : "desc");
@@ -134,8 +138,9 @@ public class CreateIndexNode extends UnaryNode implements Cloneable {
@Override
public String toString() {
- return "CreateIndex (indexName=" + indexName + ", indexPath=" + indexPath + ", type=" + indexType.name() +
- ", isUnique=" + isUnique + ", " + getSortSpecString() + ")";
+ return "CreateIndex (indexName=" + indexMeta.getIndexName() + ", indexPath=" + indexMeta.getIndexPath() +
+ ", type=" + indexMeta.getIndexMethod().name() +
+ ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java
new file mode 100644
index 0000000..da7018a
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.logical;
+
+import com.google.common.base.Objects;
+import org.apache.tajo.plan.PlanString;
+
+public class DropIndexNode extends LogicalNode implements Cloneable {
+ private String indexName;
+
+ public DropIndexNode(int pid) {
+ super(pid, NodeType.DROP_INDEX);
+ }
+
+ public void init(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(indexName);
+ }
+
+ @Override
+ public int childNum() {
+ return 0;
+ }
+
+ @Override
+ public LogicalNode getChild(int idx) {
+ return null;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj instanceof DropIndexNode) {
+ DropIndexNode other = (DropIndexNode) obj;
+ return super.equals(other) &&
+ this.indexName.equals(other.indexName);
+ }
+ return false;
+ }
+
+ @Override
+ public void preOrder(LogicalNodeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public void postOrder(LogicalNodeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public PlanString getPlanString() {
+ return new PlanString(this);
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ DropIndexNode clone = (DropIndexNode) super.clone();
+ clone.indexName = this.indexName;
+ return clone;
+ }
+
+ @Override
+ public String toString() {
+ return "DROP INDEX " + indexName;
+ }
+
+ public void setIndexName(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
index 0d59733..a36e982 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java
@@ -21,56 +21,54 @@ package org.apache.tajo.plan.logical;
import com.google.gson.Gson;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SortSpec;
-import org.apache.tajo.datum.Datum;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.plan.serder.PlanGsonHelper;
+import org.apache.tajo.util.TUtil;
+
+import java.net.URI;
public class IndexScanNode extends ScanNode {
- @Expose private SortSpec [] sortKeys;
@Expose private Schema keySchema = null;
- @Expose private Datum[] datum = null;
+ @Expose private URI indexPath = null;
+ @Expose private SimplePredicate[] predicates = null;
+
+ public IndexScanNode(int pid) {
+ super(pid);
+ setType(NodeType.INDEX_SCAN);
+ }
public IndexScanNode(int pid, ScanNode scanNode ,
- Schema keySchema , Datum[] datum, SortSpec[] sortKeys ) {
- super(pid);
+ Schema keySchema , SimplePredicate[] predicates, URI indexPath) {
+ this(pid);
init(scanNode.getTableDesc());
setQual(scanNode.getQual());
setInSchema(scanNode.getInSchema());
setTargets(scanNode.getTargets());
- setType(NodeType.BST_INDEX_SCAN);
- this.sortKeys = sortKeys;
- this.keySchema = keySchema;
- this.datum = datum;
+ this.set(keySchema, predicates, indexPath);
}
-
- public SortSpec[] getSortKeys() {
- return this.sortKeys;
+
+ public void set(Schema keySchema, SimplePredicate[] predicates, URI indexPath) {
+ this.keySchema = keySchema;
+ this.indexPath = indexPath;
+ this.predicates = predicates;
}
public Schema getKeySchema() {
return this.keySchema;
}
-
- public Datum[] getDatum() {
- return this.datum;
- }
-
- public void setSortKeys(SortSpec[] sortKeys) {
- this.sortKeys = sortKeys;
+
+ public SimplePredicate[] getPredicates() {
+ return predicates;
}
- public void setKeySchema( Schema keySchema ) {
- this.keySchema = keySchema;
- }
-
@Override
public String toString() {
Gson gson = PlanGsonHelper.getInstance();
StringBuilder builder = new StringBuilder();
builder.append("IndexScanNode : {\n");
+ builder.append(" \"indexPath\" : \"" + gson.toJson(this.indexPath) + "\"\n");
builder.append(" \"keySchema\" : \"" + gson.toJson(this.keySchema) + "\"\n");
- builder.append(" \"sortKeys\" : \"" + gson.toJson(this.sortKeys) + " \"\n");
- builder.append(" \"datums\" : \"" + gson.toJson(this.datum) + "\"\n");
+ builder.append(" \"keySortSpecs\" : \"" + gson.toJson(predicates) + " \"\n");
builder.append(" <<\"superClass\" : " + super.toString());
builder.append(">>}");
builder.append("}");
@@ -81,25 +79,12 @@ public class IndexScanNode extends ScanNode {
public boolean equals(Object obj) {
if (obj instanceof IndexScanNode) {
IndexScanNode other = (IndexScanNode) obj;
-
boolean eq = super.equals(other);
- eq = eq && this.sortKeys.length == other.sortKeys.length;
- if(eq) {
- for(int i = 0 ; i < this.sortKeys.length ; i ++) {
- eq = eq && this.sortKeys[i].getSortKey().equals(
- other.sortKeys[i].getSortKey());
- eq = eq && this.sortKeys[i].isAscending()
- == other.sortKeys[i].isAscending();
- eq = eq && this.sortKeys[i].isNullFirst()
- == other.sortKeys[i].isNullFirst();
- }
- }
- if(eq) {
- for(int i = 0 ; i < this.datum.length ; i ++ ) {
- eq = eq && this.datum[i].equals(other.datum[i]);
- }
- }
- return eq;
+ eq &= this.indexPath.equals(other.indexPath);
+ eq &= TUtil.checkEquals(this.predicates, other.predicates);
+ eq &= this.keySchema.equals(other.keySchema);
+
+ return eq;
}
return false;
}
@@ -108,15 +93,16 @@ public class IndexScanNode extends ScanNode {
public Object clone() throws CloneNotSupportedException {
IndexScanNode indexNode = (IndexScanNode) super.clone();
indexNode.keySchema = (Schema) this.keySchema.clone();
- indexNode.sortKeys = new SortSpec[this.sortKeys.length];
- for(int i = 0 ; i < sortKeys.length ; i ++ )
- indexNode.sortKeys[i] = (SortSpec) this.sortKeys[i].clone();
- indexNode.datum = new Datum[this.datum.length];
- for(int i = 0 ; i < datum.length ; i ++ ) {
- indexNode.datum[i] = this.datum[i];
- }
+ indexNode.predicates = new SimplePredicate[this.predicates.length];
+ for(int i = 0 ; i < this.predicates.length ; i ++ )
+ indexNode.predicates[i] = (SimplePredicate) this.predicates[i].clone();
+ indexNode.indexPath = this.indexPath;
return indexNode;
}
+
+ public URI getIndexPath() {
+ return indexPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java
index c1ff7e1..1ebea8b 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java
@@ -45,7 +45,7 @@ public enum NodeType {
TABLE_SUBQUERY(TableSubQueryNode.class),
SCAN(ScanNode.class),
PARTITIONS_SCAN(PartitionedTableScanNode.class),
- BST_INDEX_SCAN(IndexScanNode.class),
+ INDEX_SCAN(IndexScanNode.class),
STORE(StoreTableNode.class),
INSERT(InsertNode.class),
@@ -56,6 +56,7 @@ public enum NodeType {
ALTER_TABLESPACE (AlterTablespaceNode.class),
ALTER_TABLE (AlterTableNode.class),
CREATE_INDEX(CreateIndexNode.class),
+ DROP_INDEX(DropIndexNode.class),
TRUNCATE_TABLE (TruncateTableNode.class);
private final Class<? extends LogicalNode> baseClass;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
index 19c254b..6b7e32c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java
@@ -20,7 +20,6 @@ package org.apache.tajo.plan.rewrite;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.OverridableConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
@@ -69,18 +68,19 @@ public class BaseLogicalPlanRewriteEngine implements LogicalPlanRewriteEngine {
/**
* Rewrite a logical plan with all query rewrite rules added to this engine.
*
- * @param plan The plan to be rewritten with all query rewrite rule.
+ * @param context
* @return The rewritten plan.
*/
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
LogicalPlanRewriteRule rule;
+ LogicalPlan plan = null;
for (Entry<String, LogicalPlanRewriteRule> rewriteRule : rewriteRules.entrySet()) {
rule = rewriteRule.getValue();
- if (rule.isEligible(queryContext, plan)) {
- plan = rule.rewrite(queryContext, plan);
+ if (rule.isEligible(context)) {
if (LOG.isDebugEnabled()) {
LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query.");
}
+ plan = rule.rewrite(context);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java
index eb96149..dcfd6bf 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java
@@ -19,6 +19,7 @@
package org.apache.tajo.plan.rewrite;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.plan.rewrite.rules.AccessPathRewriter;
import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule;
import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter;
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
@@ -52,7 +53,8 @@ public class BaseLogicalPlanRewriteRuleProvider extends LogicalPlanRewriteRulePr
public Collection<Class<? extends LogicalPlanRewriteRule>> getPostRules() {
List<Class<? extends LogicalPlanRewriteRule>> rules = TUtil.newList(
ProjectionPushDownRule.class,
- PartitionedTableRewriter.class
+ PartitionedTableRewriter.class,
+ AccessPathRewriter.class
);
return rules;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java
index 267d651..f264977 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java
@@ -18,7 +18,6 @@
package org.apache.tajo.plan.rewrite;
-import org.apache.tajo.OverridableConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
@@ -29,5 +28,5 @@ public interface LogicalPlanRewriteEngine {
* @param plan The plan to be rewritten with all query rewrite rule.
* @return The rewritten plan.
*/
- LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException;
+ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java
index 2f0652b..1569a07 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java
@@ -18,7 +18,6 @@
package org.apache.tajo.plan.rewrite;
-import org.apache.tajo.OverridableConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
@@ -43,7 +42,7 @@ public interface LogicalPlanRewriteRule {
* @param plan The plan to be checked
* @return True if this rule can be applied to a given plan. Otherwise, false.
*/
- boolean isEligible(OverridableConf queryContext, LogicalPlan plan);
+ boolean isEligible(LogicalPlanRewriteRuleContext context);
/**
* Updates a logical plan and returns an updated logical plan rewritten by this rule.
@@ -53,5 +52,5 @@ public interface LogicalPlanRewriteRule {
* @param plan Input logical plan. It will not be modified.
* @return The rewritten logical plan.
*/
- LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException;
+ LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
new file mode 100644
index 0000000..6c43112
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite;
+
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.catalog.CatalogService;
+import org.apache.tajo.plan.LogicalPlan;
+
+public class LogicalPlanRewriteRuleContext {
+
+ private OverridableConf queryContext;
+ private LogicalPlan plan;
+ private CatalogService catalog;
+
+ public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan) {
+ setQueryContext(queryContext);
+ setPlan(plan);
+ }
+
+ public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan, CatalogService catalog) {
+ setQueryContext(queryContext);
+ setPlan(plan);
+ setCatalog(catalog);
+ }
+
+ public void setCatalog(CatalogService catalog) {
+ this.catalog = catalog;
+ }
+
+ public CatalogService getCatalog() {
+ return catalog;
+ }
+
+ public OverridableConf getQueryContext() {
+ return queryContext;
+ }
+
+ public void setQueryContext(OverridableConf queryContext) {
+ this.queryContext = queryContext;
+ }
+
+ public LogicalPlan getPlan() {
+ return plan;
+ }
+
+ public void setPlan(LogicalPlan plan) {
+ this.plan = plan;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java
new file mode 100644
index 0000000..477ccaf
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite.rules;
+
+import org.apache.tajo.catalog.statistics.TableStats;
+
+public abstract class AccessPathInfo {
+ public enum ScanTypeControl {
+ INDEX_SCAN,
+ SEQ_SCAN
+ }
+
+ private ScanTypeControl scanType;
+ private TableStats tableStats;
+
+ public AccessPathInfo(ScanTypeControl scanType, TableStats tableStats) {
+ this.scanType = scanType;
+ this.tableStats = tableStats;
+ }
+
+ public ScanTypeControl getScanType() {
+ return scanType;
+ }
+
+ public TableStats getTableStats() {
+ return tableStats;
+ }
+
+ public void setTableStats(TableStats tableStats) {
+ this.tableStats = tableStats;
+ }
+
+ public boolean hasTableStats() {
+ return this.tableStats != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
new file mode 100644
index 0000000..bfa8d04
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite.rules;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tajo.OverridableConf;
+import org.apache.tajo.SessionVars;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.PlanningException;
+import org.apache.tajo.plan.logical.IndexScanNode;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.plan.logical.RelationNode;
+import org.apache.tajo.plan.logical.ScanNode;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
+import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
+
+import java.util.List;
+import java.util.Stack;
+
+public class AccessPathRewriter implements LogicalPlanRewriteRule {
+ private static final Log LOG = LogFactory.getLog(AccessPathRewriter.class);
+
+ private static final String NAME = "Access Path Rewriter";
+ private Rewriter rewriter = new Rewriter();
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
+ if (context.getQueryContext().getBool(SessionVars.INDEX_ENABLED)) {
+ for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) {
+ for (RelationNode relationNode : block.getRelations()) {
+ List<AccessPathInfo> accessPathInfos = block.getAccessInfos(relationNode);
+ // If there are any alternative access paths
+ if (accessPathInfos.size() > 1) {
+ for (AccessPathInfo accessPathInfo : accessPathInfos) {
+ if (accessPathInfo.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) {
+ return true;
+ }
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
+ LogicalPlan plan = context.getPlan();
+ LogicalPlan.QueryBlock rootBlock = plan.getRootBlock();
+ rewriter.init(context.getQueryContext());
+ rewriter.visit(rootBlock, plan, rootBlock, rootBlock.getRoot(), new Stack<LogicalNode>());
+ return plan;
+ }
+
+ private final class Rewriter extends BasicLogicalPlanVisitor<Object, Object> {
+
+ private OverridableConf conf;
+
+ public void init(OverridableConf conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode,
+ Stack<LogicalNode> stack) throws PlanningException {
+ List<AccessPathInfo> accessPaths = block.getAccessInfos(scanNode);
+ AccessPathInfo optimalPath = null;
+ // initialize
+ for (AccessPathInfo accessPath : accessPaths) {
+ if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.SEQ_SCAN) {
+ optimalPath = accessPath;
+ break;
+ }
+ }
+ // find the optimal path
+ for (AccessPathInfo accessPath : accessPaths) {
+ if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) {
+ // estimation selectivity and choose the better path
+ // TODO: improve the selectivity estimation
+ double estimateSelectivity = 0.001;
+ double selectivityThreshold = conf.getFloat(SessionVars.INDEX_SELECTIVITY_THRESHOLD);
+ LOG.info("Selectivity threshold: " + selectivityThreshold);
+ LOG.info("Estimated selectivity: " + estimateSelectivity);
+ if (estimateSelectivity < selectivityThreshold) {
+ // if the estimated selectivity is greater than threshold, use the index scan
+ optimalPath = accessPath;
+ }
+ }
+ }
+
+ if (optimalPath != null && optimalPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) {
+ IndexScanInfo indexScanInfo = (IndexScanInfo) optimalPath;
+ plan.addHistory("AccessPathRewriter chooses the index scan for " + scanNode.getTableName());
+ IndexScanNode indexScanNode = new IndexScanNode(plan.newPID(), scanNode, indexScanInfo.getKeySchema(),
+ indexScanInfo.getPredicates(), indexScanInfo.getIndexPath());
+ if (stack.empty() || block.getRoot().equals(scanNode)) {
+ block.setRoot(indexScanNode);
+ } else {
+ PlannerUtil.replaceNode(plan, stack.peek(), scanNode, indexScanNode);
+ }
+ }
+ return null;
+ }
+ }
+}
[4/5] tajo git commit: TAJO-838: Improve query planner to utilize
index. (jihoon)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
index 2d35496..2edb4c4 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java
@@ -26,14 +26,17 @@ import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.CatalogConstants;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.FunctionDesc;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.exception.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
@@ -61,7 +64,6 @@ public class MemStore implements CatalogStore {
public MemStore(Configuration conf) {
}
-
public void close() throws IOException {
databases.clear();
@@ -147,6 +149,8 @@ public class MemStore implements CatalogStore {
}
databases.put(databaseName, new HashMap<String, CatalogProtos.TableDescProto>());
+ indexes.put(databaseName, new HashMap<String, IndexDescProto>());
+ indexesByColumn.put(databaseName, new HashMap<String, IndexDescProto>());
}
@Override
@@ -160,6 +164,8 @@ public class MemStore implements CatalogStore {
throw new NoSuchDatabaseException(databaseName);
}
databases.remove(databaseName);
+ indexes.remove(databaseName);
+ indexesByColumn.remove(databaseName);
}
@Override
@@ -539,17 +545,23 @@ public class MemStore implements CatalogStore {
@Override
public void createIndex(IndexDescProto proto) throws CatalogException {
final String databaseName = proto.getTableIdentifier().getDatabaseName();
+ final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName());
Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName);
Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
+ TableDescProto tableDescProto = getTable(databaseName, tableName);
- if (index.containsKey(proto.getName())) {
- throw new AlreadyExistsIndexException(proto.getName());
+ if (index.containsKey(proto.getIndexName())) {
+ throw new AlreadyExistsIndexException(proto.getIndexName());
}
- index.put(proto.getName(), proto);
- indexByColumn.put(proto.getTableIdentifier().getTableName() + "."
- + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto);
+ index.put(proto.getIndexName(), proto);
+ String originalTableName = proto.getTableIdentifier().getTableName();
+ String simpleTableName = CatalogUtil.extractSimpleName(originalTableName);
+ indexByColumn.put(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(),
+ simpleTableName,
+ getUnifiedNameForIndexByColumn(proto)),
+ proto);
}
/* (non-Javadoc)
@@ -558,10 +570,19 @@ public class MemStore implements CatalogStore {
@Override
public void dropIndex(String databaseName, String indexName) throws CatalogException {
Map<String, IndexDescProto> index = checkAndGetDatabaseNS(indexes, databaseName);
+ Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
if (!index.containsKey(indexName)) {
throw new NoSuchIndexException(indexName);
}
+ IndexDescProto proto = index.get(indexName);
+ final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName());
+ TableDescProto tableDescProto = getTable(databaseName, tableName);
index.remove(indexName);
+ String originalTableName = proto.getTableIdentifier().getTableName();
+ String simpleTableName = CatalogUtil.extractSimpleName(originalTableName);
+ indexByColumn.remove(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(),
+ simpleTableName,
+ getUnifiedNameForIndexByColumn(proto)));
}
/* (non-Javadoc)
@@ -577,19 +598,18 @@ public class MemStore implements CatalogStore {
return index.get(indexName);
}
- /* (non-Javadoc)
- * @see CatalogStore#getIndexByName(java.lang.String, java.lang.String)
- */
@Override
- public IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException {
-
+ public IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException {
Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
- if (!indexByColumn.containsKey(columnName)) {
- throw new NoSuchIndexException(columnName);
+ String simpleTableName = CatalogUtil.extractSimpleName(tableName);
+ TableDescProto tableDescProto = getTable(databaseName, simpleTableName);
+ String qualifiedColumnName = CatalogUtil.buildFQName(databaseName, simpleTableName,
+ CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames));
+ if (!indexByColumn.containsKey(qualifiedColumnName)) {
+ throw new NoSuchIndexException(qualifiedColumnName);
}
- return indexByColumn.get(columnName);
+ return indexByColumn.get(qualifiedColumnName);
}
@Override
@@ -599,50 +619,47 @@ public class MemStore implements CatalogStore {
}
@Override
- public boolean existIndexByColumn(String databaseName, String tableName, String columnName)
- throws CatalogException {
+ public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException {
Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
- return indexByColumn.containsKey(columnName);
+ TableDescProto tableDescProto = getTable(databaseName, tableName);
+ return indexByColumn.containsKey(
+ CatalogUtil.buildFQName(databaseName, CatalogUtil.extractSimpleName(tableName),
+ CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames)));
}
@Override
- public IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException {
- List<IndexDescProto> protos = new ArrayList<IndexDescProto>();
+ public List<String> getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException {
+ List<String> indexNames = new ArrayList<String>();
Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
+ String simpleTableName = CatalogUtil.extractSimpleName(tableName);
for (IndexDescProto proto : indexByColumn.values()) {
- if (proto.getTableIdentifier().getTableName().equals(tableName)) {
- protos.add(proto);
+ if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) {
+ indexNames.add(proto.getIndexName());
}
}
- return protos.toArray(new IndexDescProto[protos.size()]);
+ return indexNames;
}
-
+
@Override
- public List<IndexProto> getAllIndexes() throws CatalogException {
- List<IndexProto> indexList = new ArrayList<CatalogProtos.IndexProto>();
- Set<String> databases = indexes.keySet();
-
- for (String databaseName: databases) {
- Map<String, IndexDescProto> indexMap = indexes.get(databaseName);
-
- for (String indexName: indexMap.keySet()) {
- IndexDescProto indexDesc = indexMap.get(indexName);
- IndexProto.Builder builder = IndexProto.newBuilder();
-
- builder.setColumnName(indexDesc.getColumn().getName());
- builder.setDataType(indexDesc.getColumn().getDataType().getType().toString());
- builder.setIndexName(indexName);
- builder.setIndexType(indexDesc.getIndexMethod().toString());
- builder.setIsAscending(indexDesc.hasIsAscending() && indexDesc.getIsAscending());
- builder.setIsClustered(indexDesc.hasIsClustered() && indexDesc.getIsClustered());
- builder.setIsUnique(indexDesc.hasIsUnique() && indexDesc.getIsUnique());
-
- indexList.add(builder.build());
+ public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException {
+ Map<String, IndexDescProto> indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName);
+ String simpleTableName = CatalogUtil.extractSimpleName(tableName);
+ for (IndexDescProto proto : indexByColumn.values()) {
+ if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) {
+ return true;
}
}
-
- return indexList;
+ return false;
+ }
+
+ @Override
+ public List<IndexDescProto> getAllIndexes() throws CatalogException {
+ List<IndexDescProto> indexDescProtos = TUtil.newList();
+ for (Map<String,IndexDescProto> indexMap : indexes.values()) {
+ indexDescProtos.addAll(indexMap.values());
+ }
+ return indexDescProtos;
}
@Override
@@ -666,4 +683,13 @@ public class MemStore implements CatalogStore {
return null;
}
+ public static String getUnifiedNameForIndexByColumn(IndexDescProto proto) {
+ StringBuilder sb = new StringBuilder();
+ for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) {
+ String[] identifiers = columnSpec.getColumn().getName().split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP);
+ sb.append(identifiers[identifiers.length-1]).append("_");
+ }
+ sb.deleteCharAt(sb.length()-1);
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
index b0ba3b9..5e19ac1 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
@@ -97,17 +97,20 @@
<tns:Object order="11" type="table" name="INDEXES">
<tns:sql><![CDATA[
CREATE TABLE INDEXES (
- DB_ID INT NOT NULL REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
- TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
- INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
- PATH VARCHAR(4096),
- IS_UNIQUE BOOLEAN NOT NULL,
- IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- CONSTRAINT C_INDEXES_PK PRIMARY KEY (DB_ID, INDEX_NAME)
+ INDEX_ID INT NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1),
+ DB_ID INT NOT NULL REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
+ TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE,
+ INDEX_NAME VARCHAR(128) NOT NULL,
+ INDEX_TYPE CHAR(32) NOT NULL,
+ PATH VARCHAR(4096),
+ COLUMN_NAMES VARCHAR(128) NOT NULL, -- array of column names
+ DATA_TYPES VARCHAR(128) NOT NULL, -- array of column types
+ ORDERS VARCHAR(128) NOT NULL, -- array of column orders
+ NULL_ORDERS VARCHAR(128) NOT NULL, -- array of null orderings
+ IS_UNIQUE BOOLEAN NOT NULL,
+ IS_CLUSTERED BOOLEAN NOT NULL,
+ CONSTRAINT INDEXES_PK PRIMARY KEY (INDEX_ID),
+ CONSTRAINT C_INDEXES_UNIQ UNIQUE (DB_ID, INDEX_NAME)
)]]>
</tns:sql>
</tns:Object>
@@ -115,7 +118,7 @@
<tns:sql><![CDATA[CREATE UNIQUE INDEX idx_indexes_pk ON INDEXES (DB_ID,index_name)]]></tns:sql>
</tns:Object>
<tns:Object order="13" type="index" name="IDX_INDEXES_COLUMNS" dependsOn="INDEXES">
- <tns:sql><![CDATA[CREATE INDEX idx_indexes_columns ON INDEXES (TID,column_name)]]></tns:sql>
+ <tns:sql><![CDATA[CREATE INDEX idx_col_names ON INDEXES (DB_ID,TID,column_names)]]></tns:sql>
</tns:Object>
<tns:Object order="14" type="table" name="STATS">
<tns:sql><![CDATA[
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
index 33bf0f5..9c7f8ba 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql
@@ -1,17 +1,19 @@
CREATE TABLE INDEXES (
+ INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
DB_ID INT NOT NULL,
TID INT NOT NULL,
INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
INDEX_TYPE CHAR(32) NOT NULL,
- PATH VARCHAR(4096),
+ PATH VARCHAR(4096) NOT NULL,
+ COLUMN_NAMES VARCHAR(256) NOT NULL, -- array of column names
+ DATA_TYPES VARCHAR(128) NOT NULL, -- array of column types
+ ORDERS VARCHAR(128) NOT NULL, -- array of column orders
+ NULL_ORDERS VARCHAR(128) NOT NULL, -- array of null orderings
IS_UNIQUE BOOLEAN NOT NULL,
IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- PRIMARY KEY (DB_ID, INDEX_NAME),
+ PRIMARY KEY (INDEX_ID),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME),
- INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME)
+ INDEX IDX_COL_NAMES (DB_ID, TID, COLUMN_NAMES)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
index 33bf0f5..9c7f8ba 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql
@@ -1,17 +1,19 @@
CREATE TABLE INDEXES (
+ INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
DB_ID INT NOT NULL,
TID INT NOT NULL,
INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
INDEX_TYPE CHAR(32) NOT NULL,
- PATH VARCHAR(4096),
+ PATH VARCHAR(4096) NOT NULL,
+ COLUMN_NAMES VARCHAR(256) NOT NULL, -- array of column names
+ DATA_TYPES VARCHAR(128) NOT NULL, -- array of column types
+ ORDERS VARCHAR(128) NOT NULL, -- array of column orders
+ NULL_ORDERS VARCHAR(128) NOT NULL, -- array of null orderings
IS_UNIQUE BOOLEAN NOT NULL,
IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- PRIMARY KEY (DB_ID, INDEX_NAME),
+ PRIMARY KEY (INDEX_ID),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME),
- INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME)
+ INDEX IDX_COL_NAMES (DB_ID, TID, COLUMN_NAMES)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
index ae31d6f..d416006 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/indexes.sql
@@ -1,14 +1,18 @@
CREATE TABLE INDEXES (
+ INDEX_ID NUMBER(10) PRIMARY KEY,
DB_ID INT NOT NULL,
TID INT NOT NULL,
INDEX_NAME VARCHAR2(128) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
- DATA_TYPE VARCHAR2(128) NOT NULL,
INDEX_TYPE CHAR(32) NOT NULL,
- IS_UNIQUE CHAR NOT NULL,
- IS_CLUSTERED CHAR NOT NULL,
- IS_ASCENDING CHAR NOT NULL,
- CONSTRAINT INDEXES_PKEY PRIMARY KEY (DB_ID, INDEX_NAME),
+ PATH VARCHAR(4096) NOT NULL,
+ COLUMN_NAMES VARCHAR(256) NOT NULL, -- array of column names
+ DATA_TYPES VARCHAR(128) NOT NULL, -- array of column types
+ ORDERS VARCHAR(128) NOT NULL, -- array of column orders
+ NULL_ORDERS VARCHAR(128) NOT NULL, -- array of null orderings
+ IS_UNIQUE BOOLEAN NOT NULL,
+ IS_CLUSTERED BOOLEAN NOT NULL,
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
+ FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE,
+ UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME),
+ INDEX IDX_COL_NAMES (DB_ID, TID, COLUMN_NAMES)
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
index 8e5cbcc..f6b2b0d 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
@@ -97,18 +97,21 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
<tns:Object name="INDEXES" type="table" order="10">
<tns:sql><![CDATA[
CREATE TABLE INDEXES (
- DB_ID INT NOT NULL,
- TID INT NOT NULL,
- INDEX_NAME VARCHAR(128) NOT NULL,
- COLUMN_NAME VARCHAR(128) NOT NULL,
- DATA_TYPE VARCHAR(128) NOT NULL,
- INDEX_TYPE CHAR(32) NOT NULL,
- IS_UNIQUE BOOLEAN NOT NULL,
- IS_CLUSTERED BOOLEAN NOT NULL,
- IS_ASCENDING BOOLEAN NOT NULL,
- CONSTRAINT INDEXES_PKEY PRIMARY KEY (DB_ID, INDEX_NAME),
- FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
- FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
+ INDEX_ID SERIAL NOT NULL PRIMARY KEY,
+ DB_ID INT NOT NULL,
+ TID INT NOT NULL,
+ INDEX_NAME VARCHAR(128) NOT NULL,
+ INDEX_TYPE CHAR(32) NOT NULL,
+ PATH VARCHAR(4096) NOT NULL,
+ COLUMN_NAMES VARCHAR(256) NOT NULL, -- array of column names
+ DATA_TYPES VARCHAR(128) NOT NULL, -- array of column types
+ ORDERS VARCHAR(128) NOT NULL, -- array of column orders
+ NULL_ORDERS VARCHAR(128) NOT NULL, -- array of null orderings
+ IS_UNIQUE BOOLEAN NOT NULL,
+ IS_CLUSTERED BOOLEAN NOT NULL,
+ CONSTRAINT INDEXES_PKEY PRIMARY KEY (INDEX_ID),
+ FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE,
+ FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE
)]]>
</tns:sql>
</tns:Object>
@@ -116,7 +119,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
<tns:sql><![CDATA[CREATE UNIQUE INDEX INDEXES_IDX_DB_ID_NAME on INDEXES (DB_ID, INDEX_NAME)]]></tns:sql>
</tns:Object>
<tns:Object name="INDEXES_IDX_TID_COLUMN_NAME" type="index" order="12" dependsOn="INDEXES">
- <tns:sql><![CDATA[CREATE INDEX INDEXES_IDX_TID_COLUMN_NAME on INDEXES (TID, COLUMN_NAME)]]></tns:sql>
+ <tns:sql><![CDATA[CREATE INDEX INDEXES_IDX_TID_COLUMN_NAME on INDEXES (DB_ID, TID, COLUMN_NAMES)]]></tns:sql>
</tns:Object>
<tns:Object name="STATS" type="table" order="13">
<tns:sql><![CDATA[
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index e0b049c..044ef61 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -25,20 +25,16 @@ import org.apache.tajo.TajoConstants;
import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary;
import org.apache.tajo.catalog.exception.CatalogException;
import org.apache.tajo.catalog.exception.NoSuchFunctionException;
-import org.apache.tajo.catalog.store.PostgreSQLStore;
-import org.apache.tajo.function.Function;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.store.DerbyStore;
-import org.apache.tajo.catalog.store.MySQLStore;
-import org.apache.tajo.catalog.store.MariaDBStore;
-import org.apache.tajo.catalog.store.OracleStore;
+import org.apache.tajo.catalog.store.*;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.function.Function;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
@@ -47,6 +43,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.*;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
@@ -383,36 +381,43 @@ public class TestCatalog {
static IndexDesc desc1;
static IndexDesc desc2;
static IndexDesc desc3;
-
- static {
- desc1 = new IndexDesc(
- "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
-
- desc2 = new IndexDesc(
- "idx_test2", new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8),
- IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false);
-
- desc3 = new IndexDesc(
- "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4),
- IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true);
- }
+ static Schema relationSchema;
public static TableDesc prepareTable() throws IOException {
- Schema schema = new Schema();
- schema.addColumn("indexed.id", Type.INT4)
- .addColumn("indexed.name", Type.TEXT)
- .addColumn("indexed.age", Type.INT4)
- .addColumn("indexed.score", Type.FLOAT8);
+ relationSchema = new Schema();
+ relationSchema.addColumn(DEFAULT_DATABASE_NAME + ".indexed.id", Type.INT4)
+ .addColumn(DEFAULT_DATABASE_NAME + ".indexed.name", Type.TEXT)
+ .addColumn(DEFAULT_DATABASE_NAME + ".indexed.age", Type.INT4)
+ .addColumn(DEFAULT_DATABASE_NAME + ".indexed.score", Type.FLOAT8);
String tableName = "indexed";
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
return new TableDesc(
- CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta,
+ CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta,
new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
}
+ public static void prepareIndexDescs() throws IOException, URISyntaxException {
+ SortSpec[] colSpecs1 = new SortSpec[1];
+ colSpecs1[0] = new SortSpec(new Column("default.indexed.id", Type.INT4), true, true);
+ desc1 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test", new URI("idx_test"), colSpecs1,
+ IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema);
+
+ SortSpec[] colSpecs2 = new SortSpec[1];
+ colSpecs2[0] = new SortSpec(new Column("default.indexed.score", Type.FLOAT8), false, false);
+ desc2 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test2", new URI("idx_test2"), colSpecs2,
+ IndexMethod.TWO_LEVEL_BIN_TREE, false, false, relationSchema);
+
+ SortSpec[] colSpecs3 = new SortSpec[1];
+ colSpecs3[0] = new SortSpec(new Column("default.indexed.id", Type.INT4), true, false);
+ desc3 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed",
+ "idx_test", new URI("idx_test"), colSpecs3,
+ IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema);
+ }
+
@Test
public void testCreateSameTables() throws IOException {
assertTrue(catalog.createDatabase("tmpdb3", TajoConstants.DEFAULT_TABLESPACE_NAME));
@@ -448,20 +453,29 @@ public class TestCatalog {
@Test
public void testAddAndDelIndex() throws Exception {
TableDesc desc = prepareTable();
+ prepareIndexDescs();
assertTrue(catalog.createTable(desc));
- assertFalse(catalog.existIndexByName("db1", desc1.getName()));
- assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id"));
+ assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
+ assertFalse(catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[]{"id"}));
catalog.createIndex(desc1);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
- assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id"));
+ assertTrue(catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[]{"id"}));
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
- assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score"));
+ assertFalse(catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[]{"score"}));
catalog.createIndex(desc2);
assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName()));
- assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score"));
+ assertTrue(catalog.existIndexByColumnNames(DEFAULT_DATABASE_NAME, "indexed", new String[]{"score"}));
+
+ Set<IndexDesc> indexDescs = TUtil.newHashSet();
+ indexDescs.add(desc1);
+ indexDescs.add(desc2);
+ indexDescs.add(desc3);
+ for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) {
+ assertTrue(indexDescs.contains(index));
+ }
catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName());
assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
index db7f981..5ecb161 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
@@ -497,7 +497,7 @@ public class TajoCli {
if (response == null) {
displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
- } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
QueryId queryId = new QueryId(response.getQueryId());
waitForQueryCompleted(queryId);
@@ -510,8 +510,8 @@ public class TajoCli {
}
}
} else {
- if (response.hasErrorMessage()) {
- displayFormatter.printErrorMessage(sout, response.getErrorMessage());
+ if (response.getResult().hasErrorMessage()) {
+ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage());
wasError = true;
}
}
@@ -534,7 +534,7 @@ public class TajoCli {
if (response == null) {
displayFormatter.printErrorMessage(sout, "response is null");
wasError = true;
- } else if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
if (response.getIsForwarded()) {
QueryId queryId = new QueryId(response.getQueryId());
waitForQueryCompleted(queryId);
@@ -546,8 +546,8 @@ public class TajoCli {
}
}
} else {
- if (response.hasErrorMessage()) {
- displayFormatter.printErrorMessage(sout, response.getErrorMessage());
+ if (response.getResult().hasErrorMessage()) {
+ displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage());
wasError = true;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index 5eebc2b..2915612 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -21,9 +21,12 @@ package org.apache.tajo.cli.tsql.commands;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.TajoConstants;
+import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
import org.apache.tajo.cli.tsql.TajoCli;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.TUtil;
@@ -51,6 +54,22 @@ public class DescTableCommand extends TajoShellCommand {
context.getOutput().println("Did not find any relation named \"" + tableName + "\"");
} else {
context.getOutput().println(toFormattedString(desc));
+ // If there exists any indexes for the table, print index information
+ if (client.hasIndexes(tableName)) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Indexes:\n");
+ for (IndexDescProto index : client.getIndexes(tableName)) {
+ sb.append("\"").append(index.getIndexName()).append("\" ");
+ sb.append(index.getIndexMethod()).append(" (");
+ for (SortSpecProto key : index.getKeySortSpecsList()) {
+ sb.append(CatalogUtil.extractSimpleName(key.getColumn().getName()));
+ sb.append(key.getAscending() ? " ASC" : " DESC");
+ sb.append(key.getNullFirst() ? " NULLS FIRST, " : " NULLS LAST, ");
+ }
+ sb.delete(sb.length()-2, sb.length()-1).append(")\n");
+ }
+ context.getOutput().println(sb.toString());
+ }
}
} else if (cmd.length == 1) {
List<String> tableList = client.getTableList(null);
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
index a36fc0e..652008c 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java
@@ -21,11 +21,13 @@ package org.apache.tajo.client;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.annotation.Nullable;
+import org.apache.tajo.catalog.IndexMeta;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import java.io.Closeable;
import java.sql.SQLException;
@@ -134,4 +136,18 @@ public interface CatalogAdminClient extends Closeable {
public TableDesc getTableDesc(final String tableName) throws ServiceException;
public List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName) throws ServiceException;
+
+ public IndexDescProto getIndex(final String indexName) throws ServiceException;
+
+ public boolean existIndex(final String indexName) throws ServiceException;
+
+ public List<IndexDescProto> getIndexes(final String tableName) throws ServiceException;
+
+ public boolean hasIndexes(final String tableName) throws ServiceException;
+
+ public IndexDescProto getIndex(final String tableName, final String[] columnNames) throws ServiceException;
+
+ public boolean existIndex(final String tableName, final String[] columnName) throws ServiceException;
+
+ public boolean dropIndex(final String indexName) throws ServiceException;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index 496161d..17fdb25 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -21,13 +21,12 @@ package org.apache.tajo.client;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.annotation.Nullable;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.ipc.ClientProtos;
+import org.apache.tajo.ipc.ClientProtos.*;
import org.apache.tajo.ipc.TajoMasterClientProtocol;
import org.apache.tajo.jdbc.SQLStates;
import org.apache.tajo.rpc.NettyClientBase;
@@ -37,7 +36,6 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
-import static org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService;
import static org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService.BlockingInterface;
public class CatalogAdminClientImpl implements CatalogAdminClient {
@@ -152,10 +150,10 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
builder.setPartition(partitionMethodDesc.getProto());
}
ClientProtos.TableResponse res = tajoMasterService.createExternalTable(null, builder.build());
- if (res.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
return CatalogUtil.newTableDesc(res.getTableDesc());
} else {
- throw new SQLException(res.getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState());
+ throw new SQLException(res.getResult().getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState());
}
}
@@ -226,10 +224,10 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
builder.setSessionId(connection.sessionId);
builder.setTableName(tableName);
ClientProtos.TableResponse res = tajoMasterService.getTableDesc(null, builder.build());
- if (res.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
return CatalogUtil.newTableDesc(res.getTableDesc());
} else {
- throw new SQLException(res.getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState());
+ throw new SQLException(res.getResult().getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState());
}
}
@@ -250,10 +248,10 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
String paramFunctionName = functionName == null ? "" : functionName;
ClientProtos.FunctionResponse res = tajoMasterService.getFunctionList(null,
connection.convertSessionedString(paramFunctionName));
- if (res.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
return res.getFunctionsList();
} else {
- throw new SQLException(res.getErrorMessage());
+ throw new SQLException(res.getResult().getErrorMessage());
}
}
@@ -261,6 +259,124 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
}
@Override
+ public IndexDescProto getIndex(final String indexName) throws ServiceException {
+ return new ServerCallable<IndexDescProto>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public IndexDescProto call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ return tajoMasterService.getIndexWithName(null,
+ connection.convertSessionedString(indexName));
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public boolean existIndex(final String indexName) throws ServiceException {
+ return new ServerCallable<Boolean>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public Boolean call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ return tajoMasterService.existIndexWithName(null,
+ connection.convertSessionedString(indexName)).getValue();
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public List<IndexDescProto> getIndexes(final String tableName) throws ServiceException {
+ return new ServerCallable<List<IndexDescProto>>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public List<IndexDescProto> call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ GetIndexesResponse response = tajoMasterService.getIndexesForTable(null,
+ connection.convertSessionedString(tableName));
+ if (response.getResult().getResultCode() == ResultCode.OK) {
+ return response.getIndexesList();
+ } else {
+ throw new SQLException(response.getResult().getErrorMessage());
+ }
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public boolean hasIndexes(final String tableName) throws ServiceException {
+ return new ServerCallable<Boolean>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public Boolean call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ return tajoMasterService.existIndexesForTable(null,
+ connection.convertSessionedString(tableName)).getValue();
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public IndexDescProto getIndex(final String tableName, final String[] columnNames) throws ServiceException {
+ return new ServerCallable<IndexDescProto>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public IndexDescProto call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ GetIndexWithColumnsRequest.Builder builder = GetIndexWithColumnsRequest.newBuilder();
+ builder.setSessionId(connection.sessionId);
+ builder.setTableName(tableName);
+ for (String eachColumnName : columnNames) {
+ builder.addColumnNames(eachColumnName);
+ }
+ GetIndexWithColumnsResponse response = tajoMasterService.getIndexWithColumns(null, builder.build());
+ if (response.getResult().getResultCode() == ResultCode.OK) {
+ return response.getIndexDesc();
+ } else {
+ throw new SQLException(response.getResult().getErrorMessage());
+ }
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public boolean existIndex(final String tableName, final String[] columnName) throws ServiceException {
+ return new ServerCallable<Boolean>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public Boolean call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ GetIndexWithColumnsRequest.Builder builder = GetIndexWithColumnsRequest.newBuilder();
+ builder.setSessionId(connection.sessionId);
+ builder.setTableName(tableName);
+ for (String eachColumnName : columnName) {
+ builder.addColumnNames(eachColumnName);
+ }
+ return tajoMasterService.existIndexWithColumns(null, builder.build()).getValue();
+ }
+ }.withRetries();
+ }
+
+ @Override
+ public boolean dropIndex(final String indexName) throws ServiceException {
+ return new ServerCallable<Boolean>(connection.connPool,
+ connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) {
+
+ @Override
+ public Boolean call(NettyClientBase client) throws Exception {
+ BlockingInterface tajoMasterService = client.getStub();
+ return tajoMasterService.dropIndex(null,
+ connection.convertSessionedString(indexName)).getValue();
+ }
+ }.withRetries();
+ }
+
+ @Override
public void close() throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
index bab3518..c643679 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java
@@ -170,7 +170,7 @@ public class QueryClientImpl implements QueryClient {
SubmitQueryResponse response = tajoMasterService.submitQuery(null, builder.build());
- if (response.getResultCode() == ResultCode.OK) {
+ if (response.getResult().getResultCode() == ResultCode.OK) {
connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars()));
}
return response;
@@ -205,11 +205,11 @@ public class QueryClientImpl implements QueryClient {
ClientProtos.SubmitQueryResponse response = executeQuery(sql);
- if (response.getResultCode() == ClientProtos.ResultCode.ERROR) {
- if (response.hasErrorMessage()) {
- throw new ServiceException(response.getErrorMessage());
- } else if (response.hasErrorTrace()) {
- throw new ServiceException(response.getErrorTrace());
+ if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) {
+ if (response.getResult().hasErrorMessage()) {
+ throw new ServiceException(response.getResult().getErrorMessage());
+ } else if (response.getResult().hasErrorTrace()) {
+ throw new ServiceException(response.getResult().getErrorTrace());
}
}
@@ -241,8 +241,8 @@ public class QueryClientImpl implements QueryClient {
ClientProtos.SubmitQueryResponse response = executeQueryWithJson(json);
- if (response.getResultCode() == ClientProtos.ResultCode.ERROR) {
- throw new ServiceException(response.getErrorTrace());
+ if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) {
+ throw new ServiceException(response.getResult().getErrorTrace());
}
QueryId queryId = new QueryId(response.getQueryId());
@@ -391,8 +391,8 @@ public class QueryClientImpl implements QueryClient {
builder.setFetchRowNum(fetchRowNum);
try {
GetQueryResultDataResponse response = tajoMasterService.getQueryResultData(null, builder.build());
- if (response.getResultCode() == ClientProtos.ResultCode.ERROR) {
- throw new ServiceException(response.getErrorTrace());
+ if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) {
+ throw new ServiceException(response.getResult().getErrorTrace());
}
return response.getResultSet();
@@ -434,12 +434,12 @@ public class QueryClientImpl implements QueryClient {
builder.setIsJson(false);
ClientProtos.UpdateQueryResponse response = tajoMasterService.updateQuery(null, builder.build());
- if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars()));
return true;
} else {
- if (response.hasErrorMessage()) {
- System.err.println("ERROR: " + response.getErrorMessage());
+ if (response.getResult().hasErrorMessage()) {
+ System.err.println("ERROR: " + response.getResult().getErrorMessage());
}
return false;
}
@@ -463,11 +463,11 @@ public class QueryClientImpl implements QueryClient {
builder.setQuery(json);
builder.setIsJson(true);
ClientProtos.UpdateQueryResponse response = tajoMasterService.updateQuery(null, builder.build());
- if (response.getResultCode() == ClientProtos.ResultCode.OK) {
+ if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) {
return true;
} else {
- if (response.hasErrorMessage()) {
- System.err.println("ERROR: " + response.getErrorMessage());
+ if (response.getResult().hasErrorMessage()) {
+ System.err.println("ERROR: " + response.getResult().getErrorMessage());
}
return false;
}
@@ -588,11 +588,11 @@ public class QueryClientImpl implements QueryClient {
TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub();
GetQueryInfoResponse res = tajoMasterService.getQueryInfo(null,builder.build());
- if (res.getResultCode() == ResultCode.OK) {
+ if (res.getResult().getResultCode() == ResultCode.OK) {
return res.getQueryInfo();
} else {
abort();
- throw new ServiceException(res.getErrorMessage());
+ throw new ServiceException(res.getResult().getErrorMessage());
}
}
}.withRetries();
@@ -618,11 +618,11 @@ public class QueryClientImpl implements QueryClient {
QueryMasterClientProtocolService.BlockingInterface queryMasterService = client.getStub();
GetQueryHistoryResponse res = queryMasterService.getQueryHistory(null,builder.build());
- if (res.getResultCode() == ResultCode.OK) {
+ if (res.getResult().getResultCode() == ResultCode.OK) {
return res.getQueryHistory();
} else {
abort();
- throw new ServiceException(res.getErrorMessage());
+ throw new ServiceException(res.getResult().getErrorMessage());
}
}
}.withRetries();
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java
index 4a38934..486f95c 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java
@@ -41,11 +41,11 @@ public class QueryStatus {
submitTime = proto.getSubmitTime();
finishTime = proto.getFinishTime();
hasResult = proto.getHasResult();
- if (proto.hasErrorMessage()) {
- errorText = proto.getErrorMessage();
+ if (proto.getResult().hasErrorMessage()) {
+ errorText = proto.getResult().getErrorMessage();
}
- if (proto.hasErrorTrace()) {
- errorTrace = proto.getErrorTrace();
+ if (proto.getResult().hasErrorTrace()) {
+ errorTrace = proto.getResult().getErrorTrace();
}
queryMasterHost = proto.getQueryMasterHost();
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
index 1bc8050..0f82ae8 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
@@ -184,11 +184,11 @@ public class SessionConnection implements Closeable {
SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request);
- if (response.getResultCode() == ResultCode.OK) {
+ if (response.getResult().getResultCode() == ResultCode.OK) {
updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars()));
return Collections.unmodifiableMap(sessionVarsCache);
} else {
- throw new ServiceException(response.getMessage());
+ throw new ServiceException(response.getResult().getErrorMessage());
}
}
}.withRetries();
@@ -207,11 +207,11 @@ public class SessionConnection implements Closeable {
SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request);
- if (response.getResultCode() == ResultCode.OK) {
+ if (response.getResult().getResultCode() == ResultCode.OK) {
updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars()));
return Collections.unmodifiableMap(sessionVarsCache);
} else {
- throw new ServiceException(response.getMessage());
+ throw new ServiceException(response.getResult().getErrorMessage());
}
}
}.withRetries();
@@ -334,7 +334,7 @@ public class SessionConnection implements Closeable {
CreateSessionResponse response = tajoMasterService.createSession(null, builder.build());
- if (response.getResultCode() == ResultCode.OK) {
+ if (response.getResult().getResultCode() == ResultCode.OK) {
sessionId = response.getSessionId();
updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars()));
@@ -343,7 +343,7 @@ public class SessionConnection implements Closeable {
}
} else {
- throw new InvalidClientSessionException(response.getMessage());
+ throw new InvalidClientSessionException(response.getResult().getErrorMessage());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
index 8eafc91..9522746 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java
@@ -26,11 +26,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.tajo.QueryId;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.annotation.ThreadSafe;
+import org.apache.tajo.catalog.IndexMeta;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.ha.HAServiceUtil;
@@ -237,4 +239,39 @@ public class TajoClientImpl extends SessionConnection implements TajoClient, Que
public List<CatalogProtos.FunctionDescProto> getFunctions(final String functionName) throws ServiceException {
return catalogClient.getFunctions(functionName);
}
+
+ @Override
+ public IndexDescProto getIndex(String indexName) throws ServiceException {
+ return catalogClient.getIndex(indexName);
+ }
+
+ @Override
+ public boolean existIndex(String indexName) throws ServiceException {
+ return catalogClient.existIndex(indexName);
+ }
+
+ @Override
+ public List<IndexDescProto> getIndexes(String tableName) throws ServiceException {
+ return catalogClient.getIndexes(tableName);
+ }
+
+ @Override
+ public boolean hasIndexes(String tableName) throws ServiceException {
+ return catalogClient.hasIndexes(tableName);
+ }
+
+ @Override
+ public IndexDescProto getIndex(String tableName, String[] columnNames) throws ServiceException {
+ return catalogClient.getIndex(tableName, columnNames);
+ }
+
+ @Override
+ public boolean existIndex(String tableName, String[] columnName) throws ServiceException {
+ return catalogClient.existIndex(tableName, columnName);
+ }
+
+ @Override
+ public boolean dropIndex(String indexName) throws ServiceException {
+ return catalogClient.dropIndex(indexName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto
index a9f5498..0454901 100644
--- a/tajo-client/src/main/proto/ClientProtos.proto
+++ b/tajo-client/src/main/proto/ClientProtos.proto
@@ -31,16 +31,21 @@ enum ResultCode {
ERROR = 1;
}
+message RequestResult {
+ required ResultCode resultCode = 1;
+ optional string errorMessage = 2;
+ optional string errorTrace = 3;
+}
+
message CreateSessionRequest {
required string username = 1;
optional string baseDatabaseName = 2;
}
message CreateSessionResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional SessionIdProto sessionId = 2;
optional KeyValueSetProto sessionVars = 3;
- optional string message = 4;
}
message UpdateSessionVariableRequest {
@@ -50,9 +55,8 @@ message UpdateSessionVariableRequest {
}
message SessionUpdateResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional KeyValueSetProto sessionVars = 2;
- optional string message = 3;
}
message SessionedStringProto {
@@ -61,9 +65,8 @@ message SessionedStringProto {
}
message ExplainQueryResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional string explain = 2;
- optional string errorMessage = 3;
}
message QueryRequest {
@@ -74,9 +77,8 @@ message QueryRequest {
}
message UpdateQueryResponse {
- required ResultCode resultCode = 1;
- optional string errorMessage = 2;
- optional KeyValueSetProto sessionVars = 3;
+ required RequestResult result = 1;
+ optional KeyValueSetProto sessionVars = 2;
}
message GetQueryResultRequest {
@@ -126,7 +128,7 @@ message SerializedResultSet {
}
message SubmitQueryResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
required QueryIdProto queryId = 2;
required string userName = 3;
optional bool isForwarded = 4 [default = false];
@@ -138,24 +140,19 @@ message SubmitQueryResponse {
optional TableDescProto tableDesc = 8;
optional int32 maxRowNum = 9;
- optional string errorMessage = 10;
- optional string errorTrace = 11;
-
- optional KeyValueSetProto sessionVars = 12;
+ optional KeyValueSetProto sessionVars = 10;
}
message GetQueryStatusResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
required QueryIdProto queryId = 2;
optional QueryState state = 3;
optional float progress = 4;
optional int64 submitTime = 5;
optional int64 finishTime = 7;
optional bool hasResult = 8;
- optional string errorMessage = 9;
- optional string errorTrace = 10;
- optional string queryMasterHost = 11;
- optional int32 queryMasterPort = 12;
+ optional string queryMasterHost = 9;
+ optional int32 queryMasterPort = 10;
}
message GetQueryResultDataRequest {
@@ -165,10 +162,8 @@ message GetQueryResultDataRequest {
}
message GetQueryResultDataResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
required SerializedResultSet resultSet = 2;
- optional string errorMessage = 3;
- optional string errorTrace = 4;
}
message GetClusterInfoRequest {
@@ -228,15 +223,13 @@ message DropTableRequest {
}
message TableResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional TableDescProto tableDesc = 2;
- optional string errorMessage = 3;
}
message FunctionResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
repeated FunctionDescProto functions = 2;
- optional string errorMessage = 3;
}
message QueryInfoProto {
@@ -289,14 +282,32 @@ message QueryHistoryProto {
}
message GetQueryHistoryResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional QueryHistoryProto queryHistory = 2;
- optional string errorMessage = 3;
}
message GetQueryInfoResponse {
- required ResultCode resultCode = 1;
+ required RequestResult result = 1;
optional QueryInfoProto queryInfo = 2;
- optional string errorMessage = 3;
}
+message CreateIndexResponse {
+ required RequestResult result = 1;
+ optional IndexDescProto indexDesc = 2;
+}
+
+message GetIndexesResponse {
+ required RequestResult result = 1;
+ repeated IndexDescProto indexes = 2;
+}
+
+message GetIndexWithColumnsRequest {
+ required SessionIdProto sessionId = 1;
+ required string tableName = 2;
+ repeated string columnNames = 3;
+}
+
+message GetIndexWithColumnsResponse {
+ required RequestResult result = 1;
+ optional IndexDescProto indexDesc = 2;
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
index 10ca268..586f9ab 100644
--- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
+++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto
@@ -68,4 +68,13 @@ service TajoMasterClientProtocolService {
rpc getTableList(GetTableListRequest) returns (GetTableListResponse);
rpc getTableDesc(GetTableDescRequest) returns (TableResponse);
rpc getFunctionList(SessionedStringProto) returns (FunctionResponse);
+
+ // Index Management APIs
+ rpc getIndexWithName(SessionedStringProto) returns (IndexDescProto);
+ rpc existIndexWithName(SessionedStringProto) returns (BoolProto);
+ rpc getIndexesForTable(SessionedStringProto) returns (GetIndexesResponse);
+ rpc existIndexesForTable(SessionedStringProto) returns (BoolProto);
+ rpc getIndexWithColumns(GetIndexWithColumnsRequest) returns (GetIndexWithColumnsResponse);
+ rpc existIndexWithColumns(GetIndexWithColumnsRequest) returns (BoolProto);
+ rpc dropIndex(SessionedStringProto) returns (BoolProto);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
index b7a5da7..d65f4c3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java
@@ -171,7 +171,7 @@ public class OverridableConf extends KeyValueSet {
}
public float getFloat(ConfigKey key) {
- return getLong(key, null);
+ return getFloat(key, null);
}
public void put(ConfigKey key, String val) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
index d87bbef..be0639e 100644
--- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@ -123,6 +123,10 @@ public enum SessionVars implements ConfigKey {
NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT),
CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT),
+ // for index
+ INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT),
+ INDEX_SELECTIVITY_THRESHOLD(ConfVars.$INDEX_SELECTIVITY_THRESHOLD, "the selectivity threshold for index scan", DEFAULT),
+
// Behavior Control ---------------------------------------------------------
ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT,
"If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT),
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index ab11ddd..9a3ec6b 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -319,6 +319,10 @@ public class TajoConf extends Configuration {
$MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite
$CODEGEN("tajo.executor.codegen.enabled", false), // Runtime code generation
+ // for index
+ $INDEX_ENABLED("tajo.query.index.enabled", false),
+ $INDEX_SELECTIVITY_THRESHOLD("tajo.query.index.selectivity.threshold", 0.05f),
+
// Client -----------------------------------------------------------------
$CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour.
@@ -498,7 +502,7 @@ public class TajoConf extends Configuration {
}
public static long getLongVar(Configuration conf, ConfVars var) {
- assert (var.valClass == Long.class || var.valClass == Integer.class);
+ assert (var.valClass == Long.class || var.valClass == Integer.class || var.valClass == Float.class);
if (var.valClass == Integer.class) {
return conf.getInt(var.varname, var.defaultIntVal);
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
index a1de860..65d795d 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
@@ -21,6 +21,8 @@ package org.apache.tajo.util;
import com.google.common.base.Objects;
import java.lang.reflect.Array;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -299,4 +301,12 @@ public class TUtil {
StackTraceElement element = ste[2 + depth];
return element.getClassName() + ":" + element.getMethodName() + "(" + element.getLineNumber() +")";
}
+
+ public static URI stringToURI(String str) {
+ try {
+ return new URI(str);
+ } catch (URISyntaxException e) {
+ throw new RuntimeException("Cannot convert " + str + " to the URI type", e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index e73680d..5754040 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -77,10 +77,19 @@ schema_statement
;
index_statement
+ : create_index_statement
+ | drop_index_statement
+ ;
+
+create_index_statement
: CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)?
LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)?
;
+drop_index_statement
+ : DROP INDEX index_name = identifier
+ ;
+
database_definition
: CREATE DATABASE (if_not_exists)? dbname = identifier
;
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java
index 79513dc..4199b00 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java
@@ -214,4 +214,11 @@ public class ExecutorPreCompiler extends BasicLogicalPlanVisitor<ExecutorPreComp
return node;
}
+
+ @Override
+ public LogicalNode visitIndexScan(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException {
+ visitScan(context, plan, block, node, stack);
+ return node;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index d0db2b0..aea5a59 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1187,7 +1187,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
- public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) {
+ public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext ctx) {
String indexName = ctx.identifier().getText();
String tableName = ctx.table_name().getText();
Relation relation = new Relation(tableName);
@@ -1223,6 +1223,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
@Override
+ public Expr visitDrop_index_statement(SQLParser.Drop_index_statementContext ctx) {
+ String indexName = ctx.identifier().getText();
+ return new DropIndex(indexName);
+ }
+
+ @Override
public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) {
return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists()));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 3218e15..17a9bb1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -30,10 +30,12 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.serder.LogicalNodeDeserializer;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
@@ -47,20 +49,18 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAg
import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.IndexUtil;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Stack;
+import java.util.*;
import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType;
@@ -233,7 +233,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
return new LimitExec(ctx, limitNode.getInSchema(),
limitNode.getOutSchema(), leftExec, limitNode);
- case BST_INDEX_SCAN:
+ case INDEX_SCAN:
IndexScanNode indexScanNode = (IndexScanNode) logicalNode;
leftExec = createIndexScanExec(ctx, indexScanNode);
return leftExec;
@@ -1187,15 +1187,8 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
List<FileFragment> fragments =
FragmentConvertor.convert(ctx.getConf(), fragmentProtos);
- String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys());
- FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf());
- Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index");
-
- TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(),
- annotation.getSortKeys());
- return new BSTIndexScanExec(ctx, annotation, fragments.get(0), new Path(indexPath, indexName),
- annotation.getKeySchema(), comp, annotation.getDatum());
-
+ return new BSTIndexScanExec(ctx, annotation, fragments.get(0), annotation.getIndexPath(),
+ annotation.getKeySchema(), annotation.getPredicates());
}
public static EnforceProperty getAlgorithmEnforceProperty(Enforcer enforcer, LogicalNode node) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 9b37a80..67f5691 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -1525,6 +1525,15 @@ public class GlobalPlanner {
}
@Override
+ public LogicalNode visitIndexScan(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block,
+ IndexScanNode node, Stack<LogicalNode> stack) throws PlanningException {
+ ExecutionBlock newBlock = context.plan.newExecutionBlock();
+ newBlock.setPlan(node);
+ context.execBlockMap.put(node.getPID(), newBlock);
+ return node;
+ }
+
+ @Override
public LogicalNode visitPartitionedTableScan(GlobalPlanContext context, LogicalPlan plan,
LogicalPlan.QueryBlock block, PartitionedTableScanNode node,
Stack<LogicalNode> stack)throws PlanningException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 6adc523..3e4c62d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@ -18,23 +18,34 @@
package org.apache.tajo.engine.planner.physical;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SortSpec;
+import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.engine.planner.Projector;
+import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.EvalNode;
-import org.apache.tajo.plan.logical.ScanNode;
+import org.apache.tajo.plan.expr.EvalTreeUtil;
+import org.apache.tajo.plan.logical.IndexScanNode;
+import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.index.bst.BSTIndex;
+import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
+import java.net.URI;
+import java.util.Set;
public class BSTIndexScanExec extends PhysicalExec {
- private ScanNode scanNode;
+ private final static Log LOG = LogFactory.getLog(BSTIndexScanExec.class);
+ private IndexScanNode scanNode;
private SeekableScanner fileScanner;
private EvalNode qual;
@@ -42,31 +53,62 @@ public class BSTIndexScanExec extends PhysicalExec {
private Projector projector;
- private Datum[] datum = null;
-
+ private Datum[] values = null;
+
private boolean initialize = true;
private float progress;
- public BSTIndexScanExec(TaskAttemptContext context, ScanNode scanNode ,
- FileFragment fragment, Path fileName , Schema keySchema,
- TupleComparator comparator , Datum[] datum) throws IOException {
+ private TableStats inputStats;
+
+ public BSTIndexScanExec(TaskAttemptContext context,
+ IndexScanNode scanNode ,
+ FileFragment fragment, URI indexPrefix , Schema keySchema,
+ SimplePredicate [] predicates) throws IOException {
super(context, scanNode.getInSchema(), scanNode.getOutSchema());
this.scanNode = scanNode;
this.qual = scanNode.getQual();
- this.datum = datum;
+
+ SortSpec[] keySortSpecs = new SortSpec[predicates.length];
+ values = new Datum[predicates.length];
+ for (int i = 0; i < predicates.length; i++) {
+ keySortSpecs[i] = predicates[i].getKeySortSpec();
+ values[i] = predicates[i].getValue();
+ }
+
+ TupleComparator comparator = new BaseTupleComparator(keySchema,
+ keySortSpecs);
+
+ Schema fileScanOutSchema = mergeSubSchemas(inSchema, keySchema, scanNode.getTargets(), qual);
this.fileScanner = StorageManager.getSeekableScanner(context.getConf(),
- scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema);
+ scanNode.getTableDesc().getMeta(), inSchema, fragment, fileScanOutSchema);
this.fileScanner.init();
this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets());
- FileSystem fs = fileName.getFileSystem(context.getConf());
- this.reader = new BSTIndex(fs.getConf()).
- getIndexReader(fileName, keySchema, comparator);
+ Path indexPath = new Path(indexPrefix.toString(), context.getUniqueKeyFromFragments());
+ this.reader = new BSTIndex(context.getConf()).
+ getIndexReader(indexPath, keySchema, comparator);
this.reader.open();
}
+ private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[] targets, EvalNode qual) {
+ Schema mergedSchema = new Schema();
+ Set<Column> qualAndTargets = TUtil.newHashSet();
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual));
+ for (Target target : targets) {
+ qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()));
+ }
+ for (Column column : originalSchema.getColumns()) {
+ if (subSchema.contains(column)
+ || qualAndTargets.contains(column)
+ || qualAndTargets.contains(column)) {
+ mergedSchema.addColumn(column);
+ }
+ }
+ return mergedSchema;
+ }
+
@Override
public void init() throws IOException {
progress = 0.0f;
@@ -76,8 +118,8 @@ public class BSTIndexScanExec extends PhysicalExec {
public Tuple next() throws IOException {
if(initialize) {
//TODO : more complicated condition
- Tuple key = new VTuple(datum.length);
- key.put(datum);
+ Tuple key = new VTuple(values.length);
+ key.put(values);
long offset = reader.find(key);
if (offset == -1) {
reader.close();
@@ -116,8 +158,11 @@ public class BSTIndexScanExec extends PhysicalExec {
return outTuple;
} else {
long offset = reader.next();
- if (offset == -1) return null;
+ if (offset == -1) {
+ return null;
+ }
else fileScanner.seek(offset);
+ return null;
}
}
}
@@ -132,6 +177,16 @@ public class BSTIndexScanExec extends PhysicalExec {
@Override
public void close() throws IOException {
IOUtils.cleanup(null, reader, fileScanner);
+ if (fileScanner != null) {
+ try {
+ TableStats stats = fileScanner.getInputStats();
+ if (stats != null) {
+ inputStats = (TableStats) stats.clone();
+ }
+ } catch (CloneNotSupportedException e) {
+ e.printStackTrace();
+ }
+ }
reader = null;
fileScanner = null;
scanNode = null;
@@ -143,4 +198,13 @@ public class BSTIndexScanExec extends PhysicalExec {
public float getProgress() {
return progress;
}
+
+ @Override
+ public TableStats getInputStats() {
+ if (fileScanner != null) {
+ return fileScanner.getInputStats();
+ } else {
+ return inputStats;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
index 72a667d..9594b58 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
@@ -58,6 +58,7 @@ public class ProjectionExec extends UnaryPhysicalExec {
}
projector.eval(tuple, outTuple);
+ outTuple.setOffset(tuple.getOffset());
return outTuple;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
index 0592217..f9db842 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SortSpec;
@@ -34,6 +35,7 @@ import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
+import java.util.Arrays;
public class StoreIndexExec extends UnaryPhysicalExec {
private static final Log LOG = LogFactory.getLog(StoreIndexExec.class);
@@ -53,7 +55,7 @@ public class StoreIndexExec extends UnaryPhysicalExec {
public void init() throws IOException {
super.init();
- SortSpec[] sortSpecs = logicalPlan.getSortSpecs();
+ SortSpec[] sortSpecs = logicalPlan.getKeySortSpecs();
indexKeys = new int[sortSpecs.length];
keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs);
@@ -64,8 +66,7 @@ public class StoreIndexExec extends UnaryPhysicalExec {
}
TajoConf conf = context.getConf();
- Path indexPath = new Path(logicalPlan.getIndexPath(), ""+context.getUniqueKeyFromFragments());
- System.out.println("exec: " + indexPath);
+ Path indexPath = new Path(logicalPlan.getIndexPath().toString(), context.getUniqueKeyFromFragments());
// TODO: Create factory using reflection
BSTIndex bst = new BSTIndex(conf);
this.comparator = new BaseTupleComparator(keySchema, sortSpecs);
@@ -78,17 +79,13 @@ public class StoreIndexExec extends UnaryPhysicalExec {
public Tuple next() throws IOException {
Tuple tuple;
Tuple keyTuple;
- Tuple prevKeyTuple = null;
long offset;
while((tuple = child.next()) != null) {
offset = tuple.getOffset();
keyTuple = new VTuple(keySchema.size());
RowStoreUtil.project(tuple, keyTuple, indexKeys);
- if (prevKeyTuple == null || !prevKeyTuple.equals(keyTuple)) {
- indexWriter.write(keyTuple, offset);
- prevKeyTuple = keyTuple;
- }
+ indexWriter.write(keyTuple, offset);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java
index 29dc845..709aa81 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java
@@ -22,6 +22,7 @@ import org.apache.tajo.OverridableConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule;
+import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
@SuppressWarnings("unused")
public class ErrorInjectionRewriter implements LogicalPlanRewriteRule {
@@ -31,12 +32,12 @@ public class ErrorInjectionRewriter implements LogicalPlanRewriteRule {
}
@Override
- public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) {
+ public boolean isEligible(LogicalPlanRewriteRuleContext context) {
return true;
}
@Override
- public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException {
+ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException {
throw new NullPointerException();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/071c5d05/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 51964f0..6ba413c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -34,7 +34,7 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.engine.parser.SQLAnalyzer;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.ipc.ClientProtos;
+import org.apache.tajo.ipc.ClientProtos.ResultCode;
import org.apache.tajo.master.TajoMaster.MasterContext;
import org.apache.tajo.master.exec.DDLExecutor;
import org.apache.tajo.master.exec.QueryExecutor;
@@ -51,6 +51,7 @@ import org.apache.tajo.plan.verifier.VerificationState;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.IPCUtil;
import java.io.IOException;
import java.sql.SQLException;
@@ -90,7 +91,8 @@ public class GlobalEngine extends AbstractService {
analyzer = new SQLAnalyzer();
preVerifier = new PreLogicalPlanVerifier(context.getCatalog());
planner = new LogicalPlanner(context.getCatalog());
- optimizer = new LogicalOptimizer(context.getConf());
+ // Access path rewriter is enabled only in QueryMasterTask
+ optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog());
annotatedPlanVerifier = new LogicalPlanVerifier(context.getConf(), context.getCatalog());
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
@@ -169,13 +171,12 @@ public class GlobalEngine extends AbstractService {
responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME));
responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
responseBuilder.setIsForwarded(true);
- responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR);
String errorMessage = t.getMessage();
if (t.getMessage() == null) {
errorMessage = t.getClass().getName();
}
- responseBuilder.setErrorMessage(errorMessage);
- responseBuilder.setErrorTrace(StringUtils.stringifyException(t));
+ responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR,
+ errorMessage, StringUtils.stringifyException(t)));
return responseBuilder.build();
}
}