You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/09 04:24:17 UTC
[10/14] kylin git commit: KYLIN-2255 cleanup in StorageEngine, pass CI
KYLIN-2255 cleanup in StorageEngine, pass CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/940d84f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/940d84f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/940d84f2
Branch: refs/heads/master-hbase1.x
Commit: 940d84f237652a35f5d7debb05115ee13a938aa5
Parents: c31c849
Author: Yang Li <li...@apache.org>
Authored: Thu Dec 8 21:40:54 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 9 06:51:48 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITCombinationTest.java | 2 --
.../apache/kylin/query/ITKylinQueryTest.java | 37 +++++++-------------
.../resources/query/sql_subquery/query02.sql | 36 +++++++++++++++++++
.../query/sql_subquery/query02.sql.disabled | 31 ----------------
.../kylin/storage/hbase/HBaseStorage.java | 8 +----
5 files changed, 50 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 84573b5..5f1e3a9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.storage.hbase.HBaseStorage;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
@@ -53,7 +52,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
public static void tearDown() {
printInfo("tearDown in ITCombinationTest");
clean();
- HBaseStorage.overwriteStorageQuery = null;
Candidate.restorePriorities();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 90324b5..d485955 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -34,7 +34,6 @@ import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import org.apache.kylin.storage.hbase.HBaseStorage;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.dbunit.dataset.ITable;
@@ -95,10 +94,6 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testTimeoutQuery() throws Exception {
- if (HBaseStorage.overwriteStorageQuery != null) {
- //v1 engine does not suit
- return;
- }
try {
Map<String, String> toggles = Maps.newHashMap();
@@ -157,7 +152,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query02.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
@@ -329,30 +324,25 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testLimitEnabled() throws Exception {
- if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work
-
- try {
- //other cubes have strange aggregation groups
- RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]");
-
- List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
- for (File sqlFile : sqlFiles) {
- runSQL(sqlFile, false, false);
- assertTrue(checkLimitEnabled());
- assertTrue(checkFinalPushDownLimit());
- }
+ try {
+ //other cubes have strange aggregation groups
+ RemoveBlackoutRealizationsRule.whiteList.add("CUBE[name=test_kylin_cube_with_slr_empty]");
- } finally {
- RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]");
+ List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
+ for (File sqlFile : sqlFiles) {
+ runSQL(sqlFile, false, false);
+ assertTrue(checkLimitEnabled());
+ assertTrue(checkFinalPushDownLimit());
}
+
+ } finally {
+ RemoveBlackoutRealizationsRule.whiteList.remove("CUBE[name=test_kylin_cube_with_slr_empty]");
}
}
@Test
public void testLimitCorrectness() throws Exception {
- if (HBaseStorage.overwriteStorageQuery == null) {//v1 query engine will not work
- execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql");
- }
+ this.execLimitAndValidate(getQueryFolderPrefix() + "src/test/resources/query/sql");
}
@Test
@@ -385,7 +375,6 @@ public class ITKylinQueryTest extends KylinTestBase {
IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
ITable kylinTable = executeQuery(kylinConn, queryName, sql, false);
String queriedVersion = String.valueOf(kylinTable.getValue(0, "version"));
-
// compare the result
Assert.assertEquals(expectVersion, queriedVersion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
new file mode 100644
index 0000000..3ecfb1b
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ week_beg_dt
+ ,sum(price) as sum_price
+FROM
+(
+ select
+ test_cal_dt.week_beg_dt
+ ,test_kylin_fact.price
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+) t
+group by week_beg_dt
+order by week_beg_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled b/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled
deleted file mode 100644
index e6751b7..0000000
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disabled
+++ /dev/null
@@ -1,31 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT
- week_beg_dt
- ,sum(price) as sum_price
-FROM
-(
- select
- test_cal_dt.week_beg_dt
- ,test_kylin_fact.price
- from test_kylin_fact
- inner join edw.test_cal_dt as test_cal_dt
- on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
-) t
-group by week_beg_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/940d84f2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 43b65cb..5e8c620 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -41,8 +41,6 @@ import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
public class HBaseStorage implements IStorage {
public final static String v2CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v2.CubeStorageQuery";
- public final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery";
- public static String overwriteStorageQuery = null;//for test case
@Override
public IStorageQuery createQuery(IRealization realization) {
@@ -52,11 +50,7 @@ public class HBaseStorage implements IStorage {
CubeInstance cubeInstance = (CubeInstance) realization;
String cubeStorageQuery;
if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
- cubeStorageQuery = v1CubeStorageQuery;
- } else if (overwriteStorageQuery != null) {
- cubeStorageQuery = overwriteStorageQuery;
- } else if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) {
- cubeStorageQuery = v1CubeStorageQuery;
+ throw new IllegalStateException("Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more");
} else {
cubeStorageQuery = v2CubeStorageQuery;//by default use v2
}