You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/16 10:15:39 UTC
kylin git commit: KYLIN-1936 limit push down only applys to
ID_SHARDED_HBASE
Repository: kylin
Updated Branches:
refs/heads/master 5860c497e -> da53a7716
KYLIN-1936 limit push down only applys to ID_SHARDED_HBASE
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da53a771
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da53a771
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da53a771
Branch: refs/heads/master
Commit: da53a7716ae42dc831f16411f04097ef743ef47c
Parents: 5860c49
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Aug 16 18:13:10 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 16 18:13:18 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 9 +++++--
.../storage/gtrecord/CubeSegmentScanner.java | 4 ++++
.../gtrecord/FetchSourceAwareIterator.java | 24 -------------------
.../storage/gtrecord/IFetchSourceAware.java | 25 --------------------
.../gtrecord/SequentialCubeTupleIterator.java | 6 +++--
.../apache/kylin/query/ITKylinQueryTest.java | 5 +---
6 files changed, 16 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 6186707..c0f3ed8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -29,9 +29,9 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
+import java.util.Map.Entry;
import javax.annotation.Nullable;
@@ -64,9 +64,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
@@ -994,6 +994,11 @@ public class CubeDesc extends RootPersistentEntity {
this.nullStrings = nullStrings;
}
+ public boolean supportsLimitPushDown() {
+ //currently only ID_SHARDED_HBASE supports limit push down
+ return getStorageType() == IStorageAware.ID_SHARDED_HBASE;
+ }
+
public int getStorageType() {
return storageType;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 83ee6c7..49dc5cf 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -97,4 +97,8 @@ public class CubeSegmentScanner implements IGTScanner {
return scanner.getScannedRowCount();
}
+ public CubeSegment getSegment() {
+ return this.cubeSeg;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java
deleted file mode 100644
index cb83819..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/FetchSourceAwareIterator.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.gtrecord;
-
-import java.util.Iterator;
-
-interface FetchSourceAwareIterator<F> extends IFetchSourceAware<F>, Iterator<F> {
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java
deleted file mode 100644
index d51ca4c..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/IFetchSourceAware.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.gtrecord;
-
-import java.util.Iterator;
-
-public interface IFetchSourceAware<E> {
- public Iterator<? extends E> getFetchSource();
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index ff7fb2b..2cff76c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -62,9 +62,11 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
for (CubeSegmentScanner scanner : scanners) {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
-
+
+ boolean supportLimitPushDown = scanners.get(0).getSegment().getCubeDesc().supportsLimitPushDown();
+
this.storagePushDownLimit = context.getStoragePushDownLimit();
- if (storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) {
+ if (!supportLimitPushDown || storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) {
//normal case
tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/da53a771/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5db4342..741dd18 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -35,7 +35,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
@@ -261,9 +260,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testLimitCorrectness() throws Exception {
- if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work
- execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql");
- }
+ execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql");
}
@Test