You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/25 10:04:19 UTC
[5/5] incubator-kylin git commit: KYLIN-860 add
ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM
KYLIN-860 add ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c2bb3a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c2bb3a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c2bb3a76
Branch: refs/heads/0.8
Commit: c2bb3a763aa9a826b15aff027e7b4c8aa36fa653
Parents: b8962c1
Author: honma <ho...@ebay.com>
Authored: Thu Jun 25 16:03:54 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Jun 25 16:03:54 2015 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/observer/AggregateRegionObserver.java | 2 +-
.../storage/hbase/coprocessor/observer/AggregationScanner.java | 4 +++-
.../storage/hbase/coprocessor/observer/ObserverBehavior.java | 3 ++-
.../storage/hbase/coprocessor/observer/ObserverEnabler.java | 4 ++--
.../hbase/coprocessor/observer/AggregateRegionObserverTest.java | 4 ++--
5 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c2bb3a76/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
index 8771bf4..cf5b8d1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserver.java
@@ -84,7 +84,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
- ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR;
+ ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
byte[] behavior = scan.getAttribute(BEHAVIOR);
if (behavior != null && behavior.length != 0) {
observerBehavior = ObserverBehavior.valueOf(new String(behavior));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c2bb3a76/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
index e72444e..30c590b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregationScanner.java
@@ -85,7 +85,9 @@ public class AggregationScanner implements RegionScanner {
MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
aggregators.aggregate(bufs, results);
- aggCache.checkMemoryUsage();
+ if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ aggCache.checkMemoryUsage();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c2bb3a76/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
index 62c5bac..ff6060d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverBehavior.java
@@ -5,5 +5,6 @@ package org.apache.kylin.storage.hbase.coprocessor.observer;
public enum ObserverBehavior {
SCAN, //only scan data, used for profiling tuple scan speed
SCAN_FILTER, //only scan+filter used,used for profiling filter speed
- SCAN_FILTER_AGGR, //default normal behavior
+ SCAN_FILTER_AGGR, //aggregate the result
+ SCAN_FILTER_AGGR_CHECKMEM, //default full operations
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c2bb3a76/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
index d9933d6..c1ef5f8 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/observer/ObserverEnabler.java
@@ -73,14 +73,14 @@ public class ObserverEnabler {
if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
return new ResultScannerAdapter(aggrScanner);
} else {
// debug/profiling purpose
String toggle = BackdoorToggles.getObserverBehavior();
if (toggle == null) {
- toggle = ObserverBehavior.SCAN_FILTER_AGGR.toString(); //default behavior
+ toggle = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c2bb3a76/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
index b9f1f6a..ba09b51 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/coprocessor/observer/AggregateRegionObserverTest.java
@@ -122,7 +122,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -171,7 +171,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner,ObserverBehavior.SCAN_FILTER_AGGR);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner,ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {