You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/15 01:31:56 UTC
kylin git commit: review of KYLIN-2436, KYILN-2437,
KYLIN-2438 and KYLIN-2443
Repository: kylin
Updated Branches:
refs/heads/master eb472a335 -> f474ac5c1
review of KYLIN-2436,KYILN-2437,KYLIN-2438 and KYLIN-2443
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f474ac5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f474ac5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f474ac5c
Branch: refs/heads/master
Commit: f474ac5c1a34170922e485f53e91b3f224c16386
Parents: eb472a3
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Feb 15 09:31:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Feb 15 09:32:18 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 8 ++++----
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 6 +++---
.../main/resources/kylin-backward-compatibility.properties | 2 +-
.../kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +-
.../hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java | 2 +-
5 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 095a53f..f2ebb80 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -88,11 +88,11 @@ kylin.storage.hbase.coprocessor-mem-gb=3
# By default kylin can spill query's intermediate results to disks when it's consuming too much memory.
# Set it to false if you want query to abort immediately in such condition.
-kylin.storage.hbase.coprocessor-spill-enabled=true
+kylin.storage.partition.aggr-spill-enabled=true
# The maximum number of bytes each coprocessor is allowed to scan.
# To allow arbitrary large scan, you can set it to 0.
-kylin.storage.hbase.coprocessor-max-scan-bytes=3221225472
+kylin.storage.partition.max-scan-bytes=3221225472
# The default coprocessor timeout is (hbase.rpc.timeout * 0.9) / 1000 seconds,
# You can set it to a smaller value. 0 means use default.
@@ -158,7 +158,7 @@ kylin.snapshot.max-mb=300
# Controls the maximum number of bytes a query is allowed to scan storage.
# The default value 0 means no limit.
-# The counterpart kylin.storage.hbase.coprocessor-max-scan-bytes sets the maximum per coprocessor.
+# The counterpart kylin.storage.partition.max-scan-bytes sets the maximum per coprocessor.
kylin.query.max-scan-bytes=0
kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF
@@ -230,4 +230,4 @@ kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
-#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file
+#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fafa96d..9c54d68 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -614,11 +614,11 @@ abstract public class KylinConfigBase implements Serializable {
}
public boolean getQueryCoprocessorSpillEnabled() {
- return Boolean.parseBoolean(this.getOptional("kylin.storage.hbase.coprocessor-spill-enabled", "true"));
+ return Boolean.parseBoolean(this.getOptional("kylin.storage.partition.aggr-spill-enabled", "true"));
}
- public long getQueryCoprocessorMaxScanBytes() {
- long value = Long.parseLong(this.getOptional("kylin.storage.hbase.coprocessor-max-scan-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
+ public long getPartitionMaxScanBytes() {
+ long value = Long.parseLong(this.getOptional("kylin.storage.partition.max-scan-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
return value > 0 ? value : Long.MAX_VALUE;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/core-common/src/main/resources/kylin-backward-compatibility.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-backward-compatibility.properties b/core-common/src/main/resources/kylin-backward-compatibility.properties
index ab9bcb1..c7d9cd7 100644
--- a/core-common/src/main/resources/kylin-backward-compatibility.properties
+++ b/core-common/src/main/resources/kylin-backward-compatibility.properties
@@ -152,7 +152,7 @@ kylin.query.transformers=kylin.query.transformers
kylin.query.cache.enabled=kylin.query.cache-enabled
kylin.query.cache.threshold.duration=kylin.query.cache-threshold-duration
kylin.query.cache.threshold.scancount=kylin.query.cache-threshold-scan-count
-kylin.query.mem.budget=kylin.storage.hbase.coprocessor-max-scan-bytes
+kylin.query.mem.budget=kylin.storage.partition.max-scan-bytes
kylin.query.ignore_unknown_function=kylin.query.ignore-unknown-function
kylin.query.dim.distinct.max=kylin.query.max-dimension-count-distinct
kylin.query.security.enabled=kylin.query.security-enabled
http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 3c01da2..abc3437 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -164,7 +164,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.setQueryId(queryId);
}
builder.setSpillEnabled(cubeSeg.getConfig().getQueryCoprocessorSpillEnabled());
- builder.setMaxScanBytes(cubeSeg.getConfig().getQueryCoprocessorMaxScanBytes());
+ builder.setMaxScanBytes(cubeSeg.getConfig().getPartitionMaxScanBytes());
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f474ac5c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index be89caa..61cf067 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -293,7 +293,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
ResourceTrackingCellListIterator cellListIterator = new ResourceTrackingCellListIterator(
allCellLists,
scanReq.getStorageScanRowNumThreshold(), // for old client (scan threshold)
- request.getMaxScanBytes() == 0 ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client
+ request.hasMaxScanBytes() ? Long.MAX_VALUE : request.getMaxScanBytes(), // for new client
scanReq.getTimeout());
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());