You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/07/08 09:20:37 UTC
[kylin] branch 2.4.x updated (7065d18 -> bf85eef)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 7065d18 minor, update doc link
new e30085c KYLIN-3391 BadQueryDetector only detect first query
new e86e910 KYLIN-3390 QueryInterceptorUtil.queryInterceptors is not thread safe
new 1bdb1c1 KYLIN-2662 fix NegativeArraySizeException in TrieDictionaryForest
new bf85eef KYLIN-3403 Use IntegerCodeSystem for date type filter
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/kylin/common/KylinConfigBase.java | 5 ++++-
...dedException.java => TooBigDictionaryException.java} | 16 ++++++++++------
.../main/java/org/apache/kylin/dict/TrieDictionary.java | 3 +++
.../org/apache/kylin/dict/TrieDictionaryBuilder.java | 2 +-
.../org/apache/kylin/dict/TrieDictionaryForest.java | 2 +-
.../apache/kylin/dict/TrieDictionaryForestBuilder.java | 17 +++++++++++++++--
.../kylin/metadata/filter/FilterCodeSystemFactory.java | 2 ++
.../resources/query/sql/{query93.sql => query112.sql} | 11 +++++------
.../kylin/query/security/QueryInterceptorUtil.java | 8 +-------
.../org/apache/kylin/rest/service/BadQueryDetector.java | 5 ++---
10 files changed, 44 insertions(+), 27 deletions(-)
copy core-common/src/main/java/org/apache/kylin/common/exceptions/{ResourceLimitExceededException.java => TooBigDictionaryException.java} (75%)
copy kylin-it/src/test/resources/query/sql/{query93.sql => query112.sql} (88%)
[kylin] 02/04: KYLIN-3390 QueryInterceptorUtil.queryInterceptors is
not thread safe
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e86e910cacfab61682b41e52c499f820270f12ad
Author: yiming.xu <10...@qq.com>
AuthorDate: Mon May 28 21:16:27 2018 +0800
KYLIN-3390 QueryInterceptorUtil.queryInterceptors is not thread safe
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../org/apache/kylin/query/security/QueryInterceptorUtil.java | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
index 6ce4c8e..3641483 100644
--- a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
@@ -26,11 +26,7 @@ import org.apache.kylin.common.util.ClassUtil;
public class QueryInterceptorUtil {
private static List<QueryInterceptor> queryInterceptors = new ArrayList<>();
-
- private static void setQueryInterceptor() {
- if (queryInterceptors.size() > 0) {
- return;
- }
+ static {
String[] classes = KylinConfig.getInstanceFromEnv().getQueryInterceptors();
for (String clz : classes) {
try {
@@ -43,8 +39,6 @@ public class QueryInterceptorUtil {
}
public static List<QueryInterceptor> getQueryInterceptors() {
- setQueryInterceptor();
return queryInterceptors;
}
-
}
\ No newline at end of file
[kylin] 01/04: KYLIN-3391 BadQueryDetector only detect first query
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e30085c5896615ce1564520e23a3144e1abcffdb
Author: yiming.xu <10...@qq.com>
AuthorDate: Mon May 28 21:15:43 2018 +0800
KYLIN-3391 BadQueryDetector only detect first query
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 5 ++++-
.../main/java/org/apache/kylin/rest/service/BadQueryDetector.java | 5 ++---
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index da56f92..66be693 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1324,9 +1324,12 @@ abstract public class KylinConfigBase implements Serializable {
public int getBadQueryDefaultAlertingSeconds() {
return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90"));
}
+ public double getBadQueryDefaultAlertingCoefficient() {
+ return Double.parseDouble(getOptional("kylin.query.timeout-seconds-coefficient", "0.5"));
+ }
public int getBadQueryDefaultDetectIntervalSeconds() {
- int time = getQueryTimeoutSeconds() / 2; // half of query timeout
+ int time =(int) (getQueryTimeoutSeconds() * getBadQueryDefaultAlertingCoefficient()); // half of query timeout
if (time == 0) {
time = 60; // 60 sec
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 51f49a7..3f3db45 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -144,6 +144,7 @@ public class BadQueryDetector extends Thread {
}
private void detectBadQuery() {
+ logger.info("Detect bad query.");
long now = System.currentTimeMillis();
ArrayList<Entry> entries = new ArrayList<Entry>(runningQueries.values());
Collections.sort(entries);
@@ -156,8 +157,6 @@ public class BadQueryDetector extends Thread {
if (runningSec >= alertRunningSec) {
notify(BadQueryEntry.ADJ_SLOW, e);
dumpStackTrace(e.thread, e.queryId);
- } else {
- break; // entries are sorted by startTime
}
}
@@ -170,7 +169,7 @@ public class BadQueryDetector extends Thread {
private void setQueryThreadInterrupted(Entry e, float runningSec) {
if (queryTimeoutSeconds != 0 && runningSec >= queryTimeoutSeconds) {
e.thread.interrupt();
- logger.error("Trying to cancel query:" + e.thread.getName());
+ logger.error("Query running "+ runningSec + "s, Trying to cancel query:" + e.thread.getName());
}
}
[kylin] 03/04: KYLIN-2662 fix NegativeArraySizeException in
TrieDictionaryForest
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1bdb1c1b853a7db85ddd06004ec56574d0d83815
Author: Li Yang <li...@apache.org>
AuthorDate: Sat May 26 14:54:28 2018 +0800
KYLIN-2662 fix NegativeArraySizeException in TrieDictionaryForest
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../exceptions/TooBigDictionaryException.java | 34 ++++++++++++++++++++++
.../java/org/apache/kylin/dict/TrieDictionary.java | 3 ++
.../apache/kylin/dict/TrieDictionaryBuilder.java | 2 +-
.../apache/kylin/dict/TrieDictionaryForest.java | 2 +-
.../kylin/dict/TrieDictionaryForestBuilder.java | 17 +++++++++--
5 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/exceptions/TooBigDictionaryException.java b/core-common/src/main/java/org/apache/kylin/common/exceptions/TooBigDictionaryException.java
new file mode 100644
index 0000000..ab08c85
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/exceptions/TooBigDictionaryException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.exceptions;
+
+/**
+ * @author TinChiWay
+ * @date 2018/4/2
+ */
+@SuppressWarnings("serial")
+public class TooBigDictionaryException extends RuntimeException {
+ public TooBigDictionaryException(String message, Exception e) {
+ super(message, e);
+ }
+
+ public TooBigDictionaryException(String message) {
+ super(message);
+ }
+}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 754f451..d531c05 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -146,6 +146,9 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
return maxValueLength;
}
+ public int getStorageSizeInBytes() {
+ return trieBytes.length;
+ }
@Override
protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 18169ca..b3440a1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -39,7 +39,7 @@ import org.apache.kylin.common.util.BytesUtil;
*/
public class TrieDictionaryBuilder<T> {
- private static final int _2GB = 2000000000;
+ public static final int _2GB = 2000000000;
public static class Node {
public byte[] part;
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 09d5bc2..4642cf4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -189,8 +189,8 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
//write tree size
headOut.writeInt(trees.size());
headOut.close();
- byte[] head = byteBuf.toByteArray();
//output
+ byte[] head = byteBuf.toByteArray();
out.writeInt(head.length);
out.write(head);
}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index 0e5e63e..29cff2e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -17,13 +17,14 @@
*/
package org.apache.kylin.dict;
+import java.util.ArrayList;
+
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.TooBigDictionaryException;
import org.apache.kylin.common.util.ByteArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-
/**
* Build a trie dictionary forest if the input values is ordered, or the forest falls back to a single trie.
*/
@@ -134,6 +135,18 @@ public class TrieDictionaryForestBuilder<T> {
byte[] valueBytes = tree.getValueBytesFromIdWithoutCache(minId);
valueDivide.add(new ByteArray(valueBytes, 0, valueBytes.length));
curOffset += (tree.getMaxId() + 1);
+
+ checkDictSize();
+ }
+
+ private void checkDictSize() {
+ // due to the limitation of resource store, no dictionary beyond 2GB is allowed
+ long size = 0;
+ for (TrieDictionary trie : trees) {
+ size += trie.getStorageSizeInBytes();
+ }
+ if (size > TrieDictionaryBuilder._2GB)
+ throw new TooBigDictionaryException("Too big dictionary, dictionary cannot be bigger than 2GB");
}
private void reset() {
[kylin] 04/04: KYLIN-3403 Use IntegerCodeSystem for date type filter
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bf85eef92549512bb336a0f1ecde80c8e117481e
Author: Yifan Zhang <ev...@gmail.com>
AuthorDate: Tue Jun 12 14:57:29 2018 +0800
KYLIN-3403 Use IntegerCodeSystem for date type filter
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../metadata/filter/FilterCodeSystemFactory.java | 2 ++
kylin-it/src/test/resources/query/sql/query112.sql | 29 ++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
index bae8cf9..cf98956 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
@@ -41,6 +41,8 @@ public class FilterCodeSystemFactory {
return codeSystemMap.get("integer");
} else if (dataType.isNumberFamily()) {
return codeSystemMap.get("decimal");
+ } else if (dataType.isDateTimeFamily()) {
+ return codeSystemMap.get("integer");
} else {
return codeSystemMap.get("string");
}
diff --git a/kylin-it/src/test/resources/query/sql/query112.sql b/kylin-it/src/test/resources/query/sql/query112.sql
new file mode 100644
index 0000000..efd0d88
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query112.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT COUNT(1) AS TRANS_CNT
+FROM test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+WHERE test_cal_dt.WEEK_BEG_DT >= '2001-09-09'
+ AND test_cal_dt.WEEK_BEG_DT <= '2018-05-16'
+
\ No newline at end of file