You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/31 13:01:29 UTC
[02/50] incubator-kylin git commit: KYLIN-512 put non strictly
filtered columns into groupby
KYLIN-512 put non strictly filtered columns into groupby
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/39df16c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/39df16c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/39df16c6
Branch: refs/heads/inverted-index
Commit: 39df16c6e16e7182caa68f74fcfadc8705ce580c
Parents: ddbab01
Author: honma <ho...@ebay.com>
Authored: Tue Jan 27 22:27:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 27 22:27:23 2015 +0800
----------------------------------------------------------------------
.../metadata/filter/CaseTupleFilter.java | 1 +
.../metadata/filter/ExtractTupleFilter.java | 1 +
.../kylinolap/metadata/filter/TupleFilter.java | 28 +++++++++++----
.../hbase/coprocessor/CoprocessorFilter.java | 38 ++++++++++++++------
.../endpoint/EndpointTupleIterator.java | 8 ++++-
5 files changed, 59 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39df16c6/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
index d40dd4a..a433f53 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/CaseTupleFilter.java
@@ -107,4 +107,5 @@ public class CaseTupleFilter extends TupleFilter {
@Override
public void deserialize(byte[] bytes) {
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39df16c6/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
index 9cfe08d..1467ee8 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
@@ -54,6 +54,7 @@ public class ExtractTupleFilter extends TupleFilter {
return true;
}
+ //askliyang
private int extractDate(String type, int inDate) {
// this shifts the epoch back to astronomical year -4800 instead of the
// start of the Christian era in year AD 1 of the proleptic Gregorian
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39df16c6/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
index fa5e624..ee9a12d 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/TupleFilter.java
@@ -15,14 +15,15 @@
*/
package com.kylinolap.metadata.filter;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.kylinolap.metadata.model.TblColRef;
import com.kylinolap.metadata.tuple.ITuple;
+import javax.jdo.annotations.Column;
+
/**
*
* @author xjiang
@@ -195,13 +196,28 @@ public abstract class TupleFilter {
if (filter == null)
return true;
- if (filter.isEvaluable() == false)
+ if (!filter.isEvaluable())
return false;
for (TupleFilter child : filter.getChildren()) {
- if (isEvaluableRecursively(child) == false)
+ if (!isEvaluableRecursively(child))
return false;
}
return true;
}
+
+ public static void collectColumns(TupleFilter filter, Set<TblColRef> columns) {
+ if (filter == null)
+ return;
+
+ if (filter instanceof ColumnTupleFilter) {
+ ColumnTupleFilter columnTupleFilter = (ColumnTupleFilter) filter;
+ columns.add(columnTupleFilter.getColumn());
+ }
+
+ for (TupleFilter child : filter.getChildren()) {
+ collectColumns(child, columns);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39df16c6/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
index ecce543..8ad2079 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -40,9 +40,15 @@ public class CoprocessorFilter {
private static class FilterDecorator implements Decorator {
private RowKeyColumnIO columnIO;
+ private Set<TblColRef> unstrictlyFilteredColumns;
public FilterDecorator(ISegment seg) {
- columnIO = new RowKeyColumnIO(seg);
+ this.columnIO = new RowKeyColumnIO(seg);
+ this.unstrictlyFilteredColumns = Sets.newHashSet();
+ }
+
+ public Set<TblColRef> getUnstrictlyFilteredColumns() {
+ return unstrictlyFilteredColumns;
}
@Override
@@ -50,14 +56,19 @@ public class CoprocessorFilter {
if (filter == null)
return null;
- if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter))
+ //askliyang
+ if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
+ TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
return ConstantTupleFilter.TRUE;
+ }
if (!(filter instanceof CompareTupleFilter))
return filter;
- if (!TupleFilter.isEvaluableRecursively(filter))
+ if (!TupleFilter.isEvaluableRecursively(filter)) {
+ TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
return ConstantTupleFilter.TRUE;
+ }
// extract ColumnFilter & ConstantFilter
CompareTupleFilter compf = (CompareTupleFilter) filter;
@@ -168,9 +179,10 @@ public class CoprocessorFilter {
public static CoprocessorFilter fromFilter(final ISegment seg, TupleFilter rootFilter) {
// translate constants into dictionary IDs via a serialize copy
- byte[] bytes = TupleFilterSerializer.serialize(rootFilter, new FilterDecorator(seg));
+ FilterDecorator filterDecorator = new FilterDecorator(seg);
+ byte[] bytes = TupleFilterSerializer.serialize(rootFilter, filterDecorator);
TupleFilter copy = TupleFilterSerializer.deserialize(bytes);
- return new CoprocessorFilter(copy);
+ return new CoprocessorFilter(copy, filterDecorator.getUnstrictlyFilteredColumns());
}
public static byte[] serialize(CoprocessorFilter o) {
@@ -179,19 +191,25 @@ public class CoprocessorFilter {
public static CoprocessorFilter deserialize(byte[] filterBytes) {
TupleFilter filter = (filterBytes == null || filterBytes.length == 0) ? null : TupleFilterSerializer.deserialize(filterBytes);
- return new CoprocessorFilter(filter);
+ return new CoprocessorFilter(filter, null);
}
// ============================================================================
+ private final TupleFilter filter;
+ private final Set<TblColRef> unstrictlyFilteredColumns;
+
+ public CoprocessorFilter(TupleFilter filter, Set<TblColRef> unstrictlyFilteredColumns) {
+ this.filter = filter;
+ this.unstrictlyFilteredColumns = unstrictlyFilteredColumns;
+ }
+
public TupleFilter getFilter() {
return filter;
}
- private final TupleFilter filter;
-
- public CoprocessorFilter(TupleFilter filter) {
- this.filter = filter;
+ public Set<TblColRef> getUnstrictlyFilteredColumns() {
+ return unstrictlyFilteredColumns;
}
public boolean evaluate(ITuple tuple) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39df16c6/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 70a5f96..b16a7ae 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -3,6 +3,7 @@ package com.kylinolap.storage.hbase.coprocessor.endpoint;
import java.io.IOException;
import java.util.*;
+import com.google.common.collect.Sets;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -73,7 +74,7 @@ public class EndpointTupleIterator implements ITupleIterator {
}
if (groupBy == null) {
- groupBy = Lists.newArrayList();
+ groupBy = Sets.newHashSet();
}
if (measures == null) {
@@ -93,6 +94,11 @@ public class EndpointTupleIterator implements ITupleIterator {
this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
this.pushedDownFilter = CoprocessorFilter.fromFilter(this.seg, rootFilter);
+
+ for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
+ groupBy.add(column);
+ }
+
this.pushedDownProjector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, groupBy);
this.pushedDownAggregators = EndpointAggregators.fromFunctions(tableRecordInfo, measures);