You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/03 06:37:31 UTC
[24/50] [abbrv] kylin git commit: KYLIN-1954 refind and fix CI
KYLIN-1954 refind and fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdad2d8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdad2d8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdad2d8d
Branch: refs/heads/1.5.x-HBase1.x
Commit: fdad2d8d3ff34b069f646794226bcb73543a9474
Parents: cc9acbc
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Aug 30 09:55:57 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 30 11:05:26 2016 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 10 +++--
.../kylin/dict/BuildInFunctionTransformer.java | 5 ++-
.../localmeta/data/data_gen_config.json | 2 +-
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 4 +-
.../apache/kylin/query/ITCombinationTest.java | 14 ++++--
.../apache/kylin/query/ITKylinQueryTest.java | 13 +++++-
.../test/resources/query/sql_like/query01.sql | 31 +++++++++++++
.../test/resources/query/sql_like/query02.sql | 22 +++++++++
.../apache/kylin/query/routing/RoutingRule.java | 2 +
.../rules/RemoveBlackoutRealizationsRule.java | 47 ++++++++++++++++++++
10 files changed, 136 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 9373a02..5aac4de 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -586,10 +586,12 @@ public class FactTableGenerator {
}
for (Integer index : differentiateColumns) {
- if (currentRowTime >= differentiateBoundary) {
- columnValues.set(index, columnValues.get(index) + "_B");
- } else {
- columnValues.set(index, columnValues.get(index) + "_A");
+ if (r.nextBoolean()) {//only change half of data
+ if (currentRowTime >= differentiateBoundary) {
+ columnValues.set(index, columnValues.get(index) + "_B");
+ } else {
+ columnValues.set(index, columnValues.get(index) + "_A");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
index 9156b67..665a7e6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
@@ -70,8 +70,11 @@ public class BuildInFunctionTransformer implements ITupleFilterTransformer {
ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
while (childIterator.hasNext()) {
TupleFilter transformed = transform(childIterator.next());
- if (transformed != null)
+ if (transformed != null) {
childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
}
}
return translated == null ? tupleFilter : translated;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
index f730058..16ba3d5 100644
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ b/examples/test_case_data/localmeta/data/data_gen_config.json
@@ -1,5 +1,5 @@
{
- "differentiateBoundary": "2013-01-01",
+ "differentiateBoundary": "2013-07-01",
"columnConfigs": [
{
"columnName": "lstg_format_name",
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index ca3a9bc..4c847bf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
- statement.setString(1, "FP-GTC");
+ statement.setString(1, "FP-GTC_A");
ResultSet rs = statement.executeQuery();
@@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
String format_name = rs.getString(1);
- Assert.assertTrue("FP-GTC".equals(format_name));
+ Assert.assertTrue("FP-GTC_A".equals(format_name));
rs.close();
statement.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index cf18b20..cbd4e44 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.junit.AfterClass;
@@ -64,13 +65,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
*/
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- // return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
- return Arrays.asList(new Object[][] { { "inner", "on", "v2" }, { "left", "on", "v1" }, { "left", "on", "v2" } });
+ return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } });
}
- public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine) throws Exception {
+ public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception {
- printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine);
+ printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes);
ITKylinQueryTest.clean();
@@ -85,6 +85,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
// unset
}
+ RemoveBlackoutRealizationsRule.blackouts.clear();
+ if (excludeViewCubes) {
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
+ }
+
if ("v1".equalsIgnoreCase(queryEngine))
HBaseStorage.overwriteStorageQuery = HBaseStorage.v1CubeStorageQuery;
else
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index dcacb06..4657a5c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -34,6 +34,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
@@ -46,7 +47,6 @@ import org.junit.Test;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
@@ -57,9 +57,12 @@ public class ITKylinQueryTest extends KylinTestBase {
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
- joinType = "left";
+ joinType = "inner";
setupAll();
+
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
@AfterClass
@@ -96,6 +99,7 @@ public class ITKylinQueryTest extends KylinTestBase {
ObserverEnabler.forceCoprocessorUnset();
HBaseMetadataTestCase.staticCleanupTestMetadata();
+ RemoveBlackoutRealizationsRule.blackouts.clear();
}
@@ -147,6 +151,11 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+ public void testLikeQuery() throws Exception {
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_like", null, true);
+ }
+
+ @Test
public void testVerifyQuery() throws Exception {
verifyResultRowCount(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql
new file mode 100644
index 0000000..e1f131d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lstg_format_name like '%BIN%'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query02.sql b/kylin-it/src/test/resources/query/sql_like/query02.sql
new file mode 100644
index 0000000..ef0b4a0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query02.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='abin_a' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and
+lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABIN_Aa'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index 11498d6..53f871d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.rules.RealizationSortRule;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +38,7 @@ public abstract class RoutingRule {
private static List<RoutingRule> rules = Lists.newLinkedList();
static {
+ rules.add(new RemoveBlackoutRealizationsRule());
rules.add(new RemoveUncapableRealizationsRule());
rules.add(new RealizationSortRule());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
new file mode 100644
index 0000000..9c3d7c9
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.routing.rules;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.RoutingRule;
+
+import com.google.common.collect.Sets;
+
+/**
+ * for IT use, exclude some cubes
+ */
+public class RemoveBlackoutRealizationsRule extends RoutingRule {
+ public static Set<String> blackouts = Sets.newHashSet();
+
+ @Override
+ public void apply(List<Candidate> candidates) {
+ for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) {
+ Candidate candidate = iterator.next();
+
+ if (blackouts.contains(candidate.getRealization().getCanonicalName())) {
+ iterator.remove();
+ }
+ }
+ }
+
+}