You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/10 09:03:36 UTC
[02/17] kylin git commit: KYLIN-2377 Add kylin client query timeout
KYLIN-2377 Add kylin client query timeout
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a
Branch: refs/heads/KYLIN-2428
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen <ka...@live.com>
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Wed Feb 8 15:51:30 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../java/org/apache/kylin/cube/CubeInstance.java | 1 +
.../kylin/metadata/realization/IRealization.java | 5 ++++-
.../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++++++-
.../storage/gtrecord/GTCubeStorageQueryBase.java | 2 ++
.../gtrecord/SequentialCubeTupleIterator.java | 5 +++++
.../apache/kylin/storage/hybrid/HybridInstance.java | 1 +
7 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements Serializable {
return udfMap;
}
+ public int getQueryTimeoutSeconds() {
+ return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ }
+
// ============================================================================
// SERVER
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return sizeRecordSize;
}
+ @Override
public KylinConfig getConfig() {
return config;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
public DataModelDesc getModel();
public Set<TblColRef> getAllColumns();
-
+
public Set<ColumnDesc> getAllColumnDescs();
public List<TblColRef> getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
public long getDateRangeEnd();
public boolean supportsLimitPushDown();
+
+ public KylinConfig getConfig();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
private int finalPushDownLimit = Integer.MAX_VALUE;
private boolean hasSort = false;
private boolean acceptPartialResult = false;
+ private long deadline;
private boolean exactAggregation = false;
private boolean needStorageAggregation = false;
@@ -123,6 +124,19 @@ public class StorageContext {
}
}
+ public long getDeadline() {
+ return this.deadline;
+ }
+
+ public void setDeadline(IRealization realization) {
+ int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000;
+ if (timeout == 0) {
+ this.deadline = Long.MAX_VALUE;
+ } else {
+ this.deadline = timeout + System.currentTimeMillis();
+ }
+ }
+
public void markSort() {
this.hasSort = true;
}
@@ -202,5 +216,5 @@ public class StorageContext {
public void setStorageQuery(IStorageQuery storageQuery) {
this.storageQuery = storageQuery;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4fcfad1..4dbdf94 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
+ // set query deadline
+ context.setDeadline(cubeInstance);
// set cautious threshold to prevent out of memory
setThresholdIfNecessary(dimensionsD, metrics, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index c621215..c6b2c6c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.gridtable.GTScanTimeoutException;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
@Override
public ITuple next() {
+ if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline()) {
+ throw new GTScanTimeoutException("Query Timeout!");
+ }
+
// prevent the big query to make the Query Server OOM
if (scanCount++ > SCAN_THRESHOLD) {
throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 4f0e446..1b113ee 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
return getType() + "[name=" + name + "]";
}
+ @Override
public KylinConfig getConfig() {
return config;
}