You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/11 05:04:05 UTC
[1/4] kylin git commit: KYLIN-2541 fix sql missing in logs for beeline
Repository: kylin
Updated Branches:
refs/heads/2.0.x 273a40989 -> fdda7610b
KYLIN-2541 fix sql missing in logs for beeline
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0c538b1a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0c538b1a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0c538b1a
Branch: refs/heads/2.0.x
Commit: 0c538b1a041f2dd7e7ba65f83bcb95a58d9dbc2a
Parents: 273a409
Author: lidongsjtu <li...@apache.org>
Authored: Tue Apr 11 12:45:48 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 11 13:01:02 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HiveCmdBuilder.java | 16 ++++++++++++----
.../kylin/common/util/HiveCmdBuilderTest.java | 1 +
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c538b1a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 5942ba9..855ee48 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -62,8 +62,9 @@ public class HiveCmdBuilder {
break;
case BEELINE:
BufferedWriter bw = null;
+ File tmpHql = null;
try {
- File tmpHql = File.createTempFile("beeline_", ".hql");
+ tmpHql = File.createTempFile("beeline_", ".hql");
bw = new BufferedWriter(new FileWriter(tmpHql));
for (String statement : statements) {
bw.write(statement);
@@ -77,13 +78,20 @@ public class HiveCmdBuilder {
buf.append(tmpHql.getAbsolutePath());
buf.append(";exit $ret_code");
- if (logger.isDebugEnabled()) {
- logger.debug("The SQL to execute in beeline: \n" + FileUtils.readFileToString(tmpHql, Charset.defaultCharset()));
- }
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
IOUtils.closeQuietly(bw);
+
+ if (tmpHql != null && logger.isDebugEnabled()) {
+ String hql = null;
+ try {
+ hql = FileUtils.readFileToString(tmpHql, Charset.defaultCharset());
+ } catch (IOException e) {
+ // ignore
+ }
+ logger.debug("The SQL to execute in beeline: \n" + hql);
+ }
}
break;
default:
http://git-wip-us.apache.org/repos/asf/kylin/blob/0c538b1a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 56a64c8..3bc34b4 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -34,6 +34,7 @@ public class HiveCmdBuilderTest {
@Before
public void setup() {
+ System.setProperty("log4j.configuration", "file:../build/conf/kylin-tools-log4j.properties");
System.setProperty("KYLIN_CONF", LocalFileMetadataTestCase.LOCALMETA_TEST_DATA);
}
[3/4] kylin git commit: KYLIN-2539 bug fix
Posted by li...@apache.org.
KYLIN-2539 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c4b1d4a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c4b1d4a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c4b1d4a5
Branch: refs/heads/2.0.x
Commit: c4b1d4a5a4e11f8c50b9957e38eba71d18bc6143
Parents: fac8285
Author: lidongsjtu <li...@apache.org>
Authored: Tue Apr 11 10:46:22 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 11 13:01:28 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 4 ----
.../main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java | 5 +++++
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c4b1d4a5/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index abac33a..08a3843 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,7 +38,6 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -149,9 +148,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set query deadline
context.setDeadline(cubeInstance);
- // optimize the filter, the optimization has to be segment-irrelevant
- new FilterOptimizeTransformer().transform(filterD);
-
logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c4b1d4a5/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index d3d78a0..3fba1ee 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -60,6 +60,7 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.DynamicTupleFilter;
import org.apache.kylin.metadata.filter.ExtractTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -325,6 +326,10 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
TupleFilter filter = this.condition.accept(visitor);
context.filter = TupleFilter.and(context.filter, filter);
+
+ // optimize the filter, the optimization has to be segment-irrelevant
+ new FilterOptimizeTransformer().transform(context.filter);
+
context.filterColumns.addAll(visitor.columnsInFilter);
}
[2/4] kylin git commit: KYLIN-2540 fix concat cascade issue
Posted by li...@apache.org.
KYLIN-2540 fix concat cascade issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fac8285a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fac8285a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fac8285a
Branch: refs/heads/2.0.x
Commit: fac8285acf7db2472b95688fe4a85ca270293c99
Parents: 0c538b1
Author: lidongsjtu <li...@apache.org>
Authored: Mon Apr 10 21:50:32 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 11 13:01:21 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 1 +
.../test_case_data/sandbox/kylin.properties | 1 +
.../src/test/resources/query/sql/query104.sql | 31 ++++++++++++++++++++
.../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++++++++++++++++++
.../org/apache/kylin/rest/util/QueryUtil.java | 15 ----------
5 files changed, 61 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 6d8422e..a2273d5 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -164,6 +164,7 @@ kylin.snapshot.max-mb=300
kylin.query.max-scan-bytes=0
kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
kylin.query.cache-enabled=true
http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 1674d82..684b4dd 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -108,6 +108,7 @@ kylin.storage.hbase.max-region-count=5
kylin.storage.hbase.hfile-size-gb=2
kylin.query.udf.massin=org.apache.kylin.query.udf.MassInUDF
+kylin.query.udf.concat=org.apache.kylin.query.udf.ConcatUDF
kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
# for test
http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/kylin-it/src/test/resources/query/sql/query104.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query104.sql b/kylin-it/src/test/resources/query/sql/query104.sql
new file mode 100644
index 0000000..bb85664
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query104.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+
+select concat(meta_categ_name, lstg_format_name) as c1, concat(meta_categ_name, 'CONST') as c2, concat(meta_categ_name, concat(test_sites.site_name, lstg_format_name)) as c3, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name not in ('Crafts','Computers'))
+ group by concat(meta_categ_name, lstg_format_name), concat(meta_categ_name, 'CONST'), concat(meta_categ_name, concat(test_sites.site_name, lstg_format_name))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
new file mode 100644
index 0000000..9c0da53
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/udf/ConcatUDF.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.udf;
+
+import org.apache.calcite.linq4j.function.Parameter;
+
+public class ConcatUDF {
+
+ public String eval(@Parameter(name = "str1") String col1, @Parameter(name = "str2") String col2) {
+ return col1 + col2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fac8285a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
index 59a4a78..26d6015 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/QueryUtil.java
@@ -97,7 +97,6 @@ public class QueryUtil {
private static final Pattern PTN_SUM_1 = Pattern.compile(S0 + "SUM" + S0 + "[(]" + S0 + "[1]" + S0 + "[)]" + S0, Pattern.CASE_INSENSITIVE);
private static final Pattern PTN_NOT_EQ = Pattern.compile(S0 + "!="+ S0, Pattern.CASE_INSENSITIVE);
private static final Pattern PTN_INTERVAL = Pattern.compile("interval" + SM + "(floor\\()([\\d\\.]+)(\\))" + SM + "(second|minute|hour|day|month|year)", Pattern.CASE_INSENSITIVE);
- private static final Pattern PTN_CONCAT = Pattern.compile("concat\\(.+?\\)");//non-greedy
private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@Override
@@ -150,20 +149,6 @@ public class QueryUtil {
sql = sql.substring(0, m.start(1)) + "'" + value + "'" + sql.substring(m.end(3));
}
- //according to https://issues.apache.org/jira/browse/CALCITE-1375,
- //{fn concat('a','b')} will succeed but concat('a','b') will fail
- StringBuilder sb = new StringBuilder();
- while (true) {
- m = PTN_CONCAT.matcher(sql);
- if (!m.find())
- break;
-
- sb.append(sql.substring(0, m.start()) + "{fn " + m.group(0) + " }");
- sql = sql.substring(m.end());
- }
- String temp = sb.toString() + sql;
- sql = "".equals(temp) ? sql : temp;
-
return sql;
}
[4/4] kylin git commit: KYLIN-2540 remove unused test
Posted by li...@apache.org.
KYLIN-2540 remove unused test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdda7610
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdda7610
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdda7610
Branch: refs/heads/2.0.x
Commit: fdda7610b6cf4a1477ecc2561d55b15b2c264f1e
Parents: c4b1d4a
Author: lidongsjtu <li...@apache.org>
Authored: Mon Apr 10 22:22:19 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 11 13:01:35 2017 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/rest/util/QueryUtilTest.java | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdda7610/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
index c00cd3f..dcd8c98 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/util/QueryUtilTest.java
@@ -51,12 +51,6 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
String s = QueryUtil.massageSql(sqlRequest);
Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s);
}
- {
- SQLRequest sqlRequest = new SQLRequest();
- sqlRequest.setSql("select concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\") concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\") ()");
- String s = QueryUtil.massageSql(sqlRequest);
- Assert.assertEquals("select {fn concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\") } {fn concat(\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\",\"TEST_KYLIN_FACT\".\"LSTG_FORMAT_NAME\") } ()", s);
- }
}
@Test