You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/18 09:31:25 UTC
[12/13] incubator-kylin git commit: KYLIN-780 query test pass!
KYLIN-780 query test pass!
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a0d8c3c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a0d8c3c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a0d8c3c0
Branch: refs/heads/KYLIN-780
Commit: a0d8c3c04dd503376ec1043a942979d8e6900593
Parents: 621a80c
Author: Yang Li <li...@apache.org>
Authored: Thu Jun 18 07:14:30 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 18 15:20:13 2015 +0800
----------------------------------------------------------------------
.../query/enumerator/LookupTableEnumerator.java | 4 ++-
.../kylin/query/relnode/OLAPAggregateRel.java | 1 +
.../kylin/query/relnode/OLAPFilterRel.java | 1 +
.../apache/kylin/query/relnode/OLAPJoinRel.java | 9 ++++---
.../kylin/query/relnode/OLAPLimitRel.java | 1 +
.../kylin/query/relnode/OLAPProjectRel.java | 27 ++++++++++++++------
.../org/apache/kylin/query/relnode/OLAPRel.java | 27 ++++++++++++--------
.../apache/kylin/query/relnode/OLAPSortRel.java | 2 +-
.../kylin/query/relnode/OLAPTableScan.java | 25 +++++++++---------
9 files changed, 61 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 9e3e81d..3ae19dd 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.enumerator;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -82,7 +83,8 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
@Override
public Object[] current() {
- return current;
+ // NOTE if without the copy, sql_lookup/query03.sql will yields messy result. Very weird coz other lookup queries are all good.
+ return Arrays.copyOf(current, current.length);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 4581561..2003bc6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -349,6 +349,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel, EnumerableRe
throw new IllegalStateException("Can't create EnumerableAggregate!", e);
}
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, enumAggRel);
return enumAggRel.implement(implementor, pref);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 7d16c8d..16f5772 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -309,6 +309,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel, EnumerableRel {
EnumerableCalc enumCalcRel = new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), getInput(), program, ImmutableList.<RelCollation> of());
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, enumCalcRel);
return enumCalcRel.implement(implementor, pref);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 6be2b7f..9a85f12 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -25,14 +25,13 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.base.Preconditions;
-
import org.apache.calcite.adapter.enumerable.EnumerableJoin;
import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
import org.apache.calcite.adapter.enumerable.PhysType;
import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
import org.apache.calcite.linq4j.tree.Blocks;
import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptPlanner;
@@ -56,6 +55,8 @@ import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.schema.OLAPTable;
+import com.google.common.base.Preconditions;
+
/**
*/
public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
@@ -227,11 +228,13 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
Result result = null;
if (this.hasSubQuery) {
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, this);
result = super.implement(implementor, pref);
} else {
PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray());
RelOptTable factTable = context.firstTableScan.getTable();
- result = implementor.result(physType, Blocks.toBlock(Expressions.call(factTable.getExpression(OLAPTable.class), "executeIndexQuery", implementor.getRootExpression(), Expressions.constant(context.id))));
+ MethodCallExpression exprCall = Expressions.call(factTable.getExpression(OLAPTable.class), "executeIndexQuery", implementor.getRootExpression(), Expressions.constant(context.id));
+ result = implementor.result(physType, Blocks.toBlock(exprCall));
}
return result;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index ac9f3b1..b496054 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -101,6 +101,7 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel, EnumerableRel {
childRel.replaceTraitSet(EnumerableConvention.INSTANCE);
EnumerableLimit enumLimit = new EnumerableLimit(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), getInput(), localOffset, localFetch);
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, enumLimit);
Result res = enumLimit.implement(implementor, pref);
childRel.replaceTraitSet(CONVENTION);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 2de43e3..b22d291 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -121,12 +121,28 @@ public class OLAPProjectRel extends Project implements OLAPRel, EnumerableRel {
String fieldName = columnField.getName();
Set<TblColRef> sourceCollector = new HashSet<TblColRef>();
TblColRef column = translateRexNode(rex, inputColumnRowType, fieldName, sourceCollector);
+ if (column == null)
+ throw new IllegalStateException("No TblColRef found in " + rex);
columns.add(column);
sourceColumns.add(sourceCollector);
}
return new ColumnRowType(columns, sourceColumns);
}
+ private TblColRef translateFirstRexInputRef(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
+ for (RexNode operand : call.getOperands()) {
+ if (operand instanceof RexInputRef) {
+ return translateRexInputRef((RexInputRef) operand, inputColumnRowType, fieldName, sourceCollector);
+ }
+ if (operand instanceof RexCall) {
+ TblColRef r = translateFirstRexInputRef((RexCall) operand, inputColumnRowType, fieldName, sourceCollector);
+ if (r != null)
+ return r;
+ }
+ }
+ return null;
+ }
+
private TblColRef translateRexNode(RexNode rexNode, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
TblColRef column = null;
if (rexNode instanceof RexInputRef) {
@@ -171,16 +187,10 @@ public class OLAPProjectRel extends Project implements OLAPRel, EnumerableRel {
private TblColRef translateRexCall(RexCall call, ColumnRowType inputColumnRowType, String fieldName, Set<TblColRef> sourceCollector) {
SqlOperator operator = call.getOperator();
if (operator == SqlStdOperatorTable.EXTRACT_DATE) {
- List<RexNode> extractDateOps = call.getOperands();
- RexCall reinterpret = (RexCall) extractDateOps.get(1);
- List<RexNode> reinterpretOps = reinterpret.getOperands();
- RexInputRef inputRef = (RexInputRef) reinterpretOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
} else if (operator instanceof SqlUserDefinedFunction) {
if (operator.getName().equals("QUARTER")) {
- List<RexNode> quaterOps = call.getOperands();
- RexInputRef inputRef = (RexInputRef) quaterOps.get(0);
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
+ return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
}
} else if (operator instanceof SqlCaseOperator) {
for (RexNode operand : call.getOperands()) {
@@ -215,6 +225,7 @@ public class OLAPProjectRel extends Project implements OLAPRel, EnumerableRel {
enumCalcRel = new EnumerableCalc(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), child, program, ImmutableList.<RelCollation> of());
}
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, enumCalcRel);
return enumCalcRel.implement(implementor, pref);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
index 806323c..974b95c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.Stack;
@@ -30,6 +31,8 @@ import org.apache.calcite.rel.RelNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
+
/**
*/
public interface OLAPRel extends RelNode {
@@ -133,26 +136,30 @@ public interface OLAPRel extends RelNode {
*/
public static class JavaImplementor extends EnumerableRelImplementor {
- private OLAPContext parentContext;
+ private IdentityHashMap<EnumerableRel, OLAPContext> parentContexts = Maps.newIdentityHashMap();
+ private boolean calciteDebug = System.getProperty("calcite.debug") != null;
public JavaImplementor(EnumerableRelImplementor enumImplementor) {
super(enumImplementor.getRexBuilder(), new LinkedHashMap<String, Object>());
}
- public OLAPContext getParentContext() {
- return parentContext;
+ public void putParentContextForTableScanChild(OLAPRel olapParent, EnumerableRel enumParent) {
+ parentContexts.put(enumParent, olapParent.getContext());
}
-
+
@Override
public EnumerableRel.Result visitChild(EnumerableRel parent, int ordinal, EnumerableRel child, EnumerableRel.Prefer prefer) {
- if (parent instanceof OLAPRel) {
- OLAPRel olapRel = (OLAPRel) parent;
- this.parentContext = olapRel.getContext();
- System.out.println("Parent: " + olapRel);
+ // OLAPTableScan is shared instance when the same table appears multiple times in the tree.
+ // Its context must be set (or corrected) right before visiting.
+ if (child instanceof OLAPTableScan) {
+ OLAPContext parentContext = parentContexts.get(parent);
+ if (parentContext != null) {
+ ((OLAPTableScan) child).overrideContext(parentContext);
+ }
}
- if (child instanceof OLAPRel) {
+ if (calciteDebug && child instanceof OLAPRel) {
OLAPRel olapRel = (OLAPRel) child;
- System.out.println(this.parentContext + " - " + olapRel.getContext() + " - " + olapRel);
+ System.out.println(olapRel.getContext() + " - " + olapRel);
}
return super.visitChild(parent, ordinal, child, prefer);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
index 2591719..5048ba2 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java
@@ -118,7 +118,7 @@ public class OLAPSortRel extends Sort implements EnumerableRel, OLAPRel {
childRel.replaceTraitSet(EnumerableConvention.INSTANCE);
EnumerableSort enumSort = new EnumerableSort(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE, collation), getInput(), collation, offset, fetch);
-
+ ((OLAPRel.JavaImplementor) implementor).putParentContextForTableScanChild(this, enumSort);
Result res = enumSort.implement(implementor, pref);
childRel.replaceTraitSet(OLAPRel.CONVENTION);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a0d8c3c0/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index f149592..afb4b60 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -28,6 +28,7 @@ import org.apache.calcite.adapter.enumerable.PhysType;
import org.apache.calcite.adapter.enumerable.PhysTypeImpl;
import org.apache.calcite.linq4j.tree.Blocks;
import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
import org.apache.calcite.linq4j.tree.Primitive;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptCost;
@@ -47,11 +48,6 @@ import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.kylin.query.schema.OLAPSchema;
-import org.apache.kylin.query.schema.OLAPTable;
-
-import com.google.common.base.Preconditions;
-
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.optrule.OLAPAggregateRule;
@@ -61,6 +57,10 @@ import org.apache.kylin.query.optrule.OLAPLimitRule;
import org.apache.kylin.query.optrule.OLAPProjectRule;
import org.apache.kylin.query.optrule.OLAPSortRule;
import org.apache.kylin.query.optrule.OLAPToEnumerableConverterRule;
+import org.apache.kylin.query.schema.OLAPSchema;
+import org.apache.kylin.query.schema.OLAPTable;
+
+import com.google.common.base.Preconditions;
/**
*/
@@ -96,6 +96,10 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
public OLAPContext getContext() {
return context;
}
+
+ void overrideContext(OLAPContext context) {
+ this.context = context;
+ }
@Override
public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
@@ -192,21 +196,16 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
throw new IllegalStateException("implementor is not JavaImplementor");
JavaImplementor javaImplementor = (JavaImplementor) implementor;
- int ctxId = this.context.id;
- if (javaImplementor.getParentContext() != null) {
- ctxId = javaImplementor.getParentContext().id;
- }
-
PhysType physType = PhysTypeImpl.of(javaImplementor.getTypeFactory(), this.rowType, pref.preferArray());
String execFunction = genExecFunc();
- return javaImplementor.result(physType, Blocks.toBlock(Expressions.call(table.getExpression(OLAPTable.class), execFunction, javaImplementor.getRootExpression(), Expressions.constant(ctxId))));
+ MethodCallExpression exprCall = Expressions.call(table.getExpression(OLAPTable.class), execFunction, javaImplementor.getRootExpression(), Expressions.constant(context.id));
+ return javaImplementor.result(physType, Blocks.toBlock(exprCall));
}
private String genExecFunc() {
- // if the table to scan is not the fact table of cube, then it's a
- // lookup table
+ // if the table to scan is not the fact table of cube, then it's a lookup table
if (context.hasJoin == false && tableName.equalsIgnoreCase(context.realization.getFactTable()) == false) {
return "executeLookupTableQuery";
} else {