You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/02 08:48:39 UTC
[1/2] incubator-kylin git commit: KYLIN-942 enable profiling backdoor
toggles on v2 storage
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 480bc0b71 -> c37bc653f
KYLIN-942 enable profiling backdoor toggles on v2 storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c37bc653
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c37bc653
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c37bc653
Branch: refs/heads/2.x-staging
Commit: c37bc653f495ae7e1804d543bba0d1fa15d95b66
Parents: d2e2d21
Author: honma <ho...@ebay.com>
Authored: Mon Nov 2 11:40:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 2 15:52:56 2015 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 15 +-
.../common/util/MemoryBudgetController.java | 1 +
.../cube/inmemcubing/InMemCubeBuilder.java | 5 +-
.../apache/kylin/gridtable/EmptyGTScanner.java | 58 +++
.../kylin/gridtable/GTAggregateScanner.java | 47 +--
.../apache/kylin/gridtable/GTScanRequest.java | 49 ++-
.../common/coprocessor/CoprocessorBehavior.java | 10 +
.../observer/AggregateRegionObserver.java | 9 +-
.../observer/AggregationScanner.java | 42 +-
.../coprocessor/observer/ObserverBehavior.java | 10 -
.../coprocessor/observer/ObserverEnabler.java | 7 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 23 +-
.../coprocessor/endpoint/CubeVisitService.java | 11 +-
.../endpoint/generated/CubeVisitProtos.java | 386 +++++++++++++------
.../endpoint/protobuf/CubeVisit.proto | 8 +-
.../observer/AggregateRegionObserverTest.java | 5 +-
16 files changed, 489 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index bac7258..1e26557 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -20,18 +20,23 @@ package org.apache.kylin.common.debug;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
*/
public class BackdoorToggles {
+ private static final Logger logger = LoggerFactory.getLogger(BackdoorToggles.class);
+
private static final ThreadLocal<Map<String, String>> _backdoorToggles = new ThreadLocal<Map<String, String>>();
public static void setToggles(Map<String, String> toggles) {
_backdoorToggles.set(toggles);
}
- public static String getObserverBehavior() {
- return getString(DEBUG_TOGGLE_OBSERVER_BEHAVIOR);
+ public static String getCoprocessorBehavior() {
+ return getString(DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR);
}
public static String getHbaseCubeQueryVersion() {
@@ -101,14 +106,14 @@ public class BackdoorToggles {
public final static String DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL = "DEBUG_TOGGLE_HBASE_CUBE_QUERY_PROTOCOL";
/**
- * set DEBUG_TOGGLE_OBSERVER_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR to control observer behavior for debug/profile usage
+ * set DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR=SCAN/SCAN_FILTER/SCAN_FILTER_AGGR/SCAN_FILTER_AGGR_CHECKMEM to control observer behavior for debug/profile usage
*
example:(put it into request body)
"backdoorToggles": {
- "DEBUG_TOGGLE_OBSERVER_BEHAVIOR": "SCAN"
+ "DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR": "SCAN"
}
*/
- public final static String DEBUG_TOGGLE_OBSERVER_BEHAVIOR = "DEBUG_TOGGLE_OBSERVER_BEHAVIOR";
+ public final static String DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR = "DEBUG_TOGGLE_COPROCESSOR_BEHAVIOR";
/**
* set DEBUG_TOGGLE_LOCAL_COPROCESSOR=true to run coprocessor at client side (not in HBase region server)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index db5996d..2c26666 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,6 +57,7 @@ public class MemoryBudgetController {
public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
public static final int ONE_MB = 1024 * 1024;
+ public static final int ONE_GB = 1024 * 1024 * 1024;
private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 84abd47..d7aa50a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kylin.common.topn.Counter;
import org.apache.kylin.common.topn.TopNCounter;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.common.util.Pair;
@@ -342,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
- GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
+ GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false);
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + baseCuboidId);
@@ -463,7 +464,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
totalSum[i] = Math.round(total);
}
-
+
}
if (totalSumForSanityCheck == null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
new file mode 100644
index 0000000..b3507c9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class EmptyGTScanner implements IGTScanner {
+ @Override
+ public GTInfo getInfo() {
+ return null;
+ }
+
+ @Override
+ public int getScannedRowCount() {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new Iterator<GTRecord>() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public GTRecord next() {
+ return null;
+ }
+
+ @Override
+ public void remove() {
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 193a05c..35c1210 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -9,10 +9,8 @@ import java.util.SortedMap;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.measure.HLLCAggregator;
-import org.apache.kylin.metadata.measure.LDCAggregator;
+import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.metadata.measure.MeasureAggregator;
-import org.apache.kylin.metadata.measure.TopNAggregator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -23,6 +21,7 @@ public class GTAggregateScanner implements IGTScanner {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(GTAggregateScanner.class);
+ private int aggregatedRowCount = 0;
final GTInfo info;
final ImmutableBitSet dimensions; // dimensions to return, can be more than group by
@@ -31,19 +30,20 @@ public class GTAggregateScanner implements IGTScanner {
final String[] metricsAggrFuncs;
final IGTScanner inputScanner;
final AggregationCache aggrCache;
+ final boolean enableMemCheck;
- public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
+ public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, boolean enableMemCheck) {
if (req.hasAggregation() == false)
throw new IllegalStateException();
this.info = inputScanner.getInfo();
- //TODO bug? what if multiple metrics on same column? and what is this?
this.dimensions = req.getColumns().andNot(req.getAggrMetrics());
this.groupBy = req.getAggrGroupBy();
this.metrics = req.getAggrMetrics();
this.metricsAggrFuncs = req.getAggrMetricsFuncs();
this.inputScanner = inputScanner;
this.aggrCache = new AggregationCache();
+ this.enableMemCheck = enableMemCheck;
}
@Override
@@ -71,11 +71,7 @@ public class GTAggregateScanner implements IGTScanner {
/** return the estimate memory size of aggregation cache */
public long getEstimateSizeOfAggrCache() {
- return aggrCache.esitmateMemSize();
- }
-
- public Object[] getTotalSumForSanityCheck() {
- return aggrCache.calculateTotalSumSanityCheck();
+ return aggrCache.estimatedMemSize();
}
class AggregationCache {
@@ -145,6 +141,12 @@ public class GTAggregateScanner implements IGTScanner {
}
void aggregate(GTRecord r) {
+ if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) {
+ if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
+ throw new RuntimeException("AggregationCache exceed 1GB");
+ }
+ }
+
final byte[] key = createKey(r);
MeasureAggregator[] aggrs = aggBufMap.get(key);
if (aggrs == null) {
@@ -162,30 +164,7 @@ public class GTAggregateScanner implements IGTScanner {
return info.codeSystem.newMetricsAggregators(metrics, metricsAggrFuncs);
}
- public Object[] calculateTotalSumSanityCheck() {
- MeasureAggregator[] totalSum = newAggregators();
-
- // skip expensive aggregation
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] instanceof HLLCAggregator || totalSum[i] instanceof LDCAggregator || totalSum[i] instanceof TopNAggregator)
- totalSum[i] = null;
- }
-
- for (MeasureAggregator[] entry : aggBufMap.values()) {
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] != null)
- totalSum[i].aggregate(entry[i].getState());
- }
- }
- Object[] result = new Object[totalSum.length];
- for (int i = 0; i < totalSum.length; i++) {
- if (totalSum[i] != null)
- result[i] = totalSum[i].getState();
- }
- return result;
- }
-
- public long esitmateMemSize() {
+ public long estimatedMemSize() {
if (aggBufMap.isEmpty())
return 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c81dd63..ba997ec 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
@@ -117,14 +118,52 @@ public class GTScanRequest {
}
public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
+ return decorateScanner(scanner, true, true, true);
+ }
+
+ /**
+ * doFilter,doAggr,doMemCheck are only for profiling use.
+ * in normal cases they are all true.
+ *
+ * Refer to CoprocessorBehavior for explanation
+ */
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean doFilter, boolean doAggr, boolean doMemCheck) throws IOException {
IGTScanner result = scanner;
- if (this.hasFilterPushDown()) {
- result = new GTFilterScanner(result, this);
+ if (!doFilter) { //Skip reading this section if you're not profiling!
+ lookAndForget(result);
+ return new EmptyGTScanner();
+ } else {
+
+ if (this.hasFilterPushDown()) {
+ result = new GTFilterScanner(result, this);
+ }
+
+ if (!doAggr) {//Skip reading this section if you're not profiling!
+ lookAndForget(result);
+ return new EmptyGTScanner();
+ }
+
+ if (this.allowPreAggregation && this.hasAggregation()) {
+ result = new GTAggregateScanner(result, this, doMemCheck);
+ }
+ return result;
}
- if (this.allowPreAggregation && this.hasAggregation()) {
- result = new GTAggregateScanner(result, this);
+ }
+
+ //touch every byte of the cell so that the cost of scanning will be trully reflected
+ private void lookAndForget(IGTScanner scanner) {
+ byte meaninglessByte = 0;
+ for (GTRecord gtRecord : scanner) {
+ for (ByteArray col : gtRecord.getInternal()) {
+ if (col != null) {
+ int endIndex = col.offset() + col.length();
+ for (int i = col.offset(); i < endIndex; ++i) {
+ meaninglessByte += col.array()[i];
+ }
+ }
+ }
}
- return result;
+ System.out.println("meaningless byte is now " + meaninglessByte);
}
public boolean hasFilterPushDown() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
new file mode 100644
index 0000000..df81177
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorBehavior.java
@@ -0,0 +1,10 @@
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+/**
+ */
+public enum CoprocessorBehavior {
+ SCAN, //only scan data, used for profiling tuple scan speed. Will not return any result
+ SCAN_FILTER, //only scan+filter used,used for profiling filter speed. Will not return any result
+ SCAN_FILTER_AGGR, //aggregate the result. Will return results
+ SCAN_FILTER_AGGR_CHECKMEM, //default full operations. Will return results
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 824a7d3..c7b650a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -84,15 +85,15 @@ public class AggregateRegionObserver extends BaseRegionObserver {
byte[] filterBytes = scan.getAttribute(FILTER);
CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
- ObserverBehavior observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ CoprocessorBehavior coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
try {
byte[] behavior = scan.getAttribute(BEHAVIOR);
if (behavior != null && behavior.length != 0) {
- observerBehavior = ObserverBehavior.valueOf(new String(behavior));
+ coprocessorBehavior = CoprocessorBehavior.valueOf(new String(behavior));
}
} catch (Exception e) {
LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
- observerBehavior = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM;
+ coprocessorBehavior = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM;
}
// start/end region operation & sync on scanner is suggested by the
@@ -102,7 +103,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
region.startRegionOperation();
try {
synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner, observerBehavior);
+ return new AggregationScanner(type, filter, projector, aggregators, innerScanner, coprocessorBehavior);
}
} finally {
region.closeRegionOperation();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index bc74ac1..fc1d967 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -38,9 +39,9 @@ import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
public class AggregationScanner implements RegionScanner {
private RegionScanner outerScanner;
- private ObserverBehavior behavior;
+ private CoprocessorBehavior behavior;
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, ObserverBehavior behavior) throws IOException {
+ public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, CoprocessorBehavior behavior) throws IOException {
AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
@@ -64,6 +65,8 @@ public class AggregationScanner implements RegionScanner {
ObserverTuple tuple = new ObserverTuple(type);
boolean hasMore = true;
List<Cell> results = new ArrayList<Cell>();
+ byte meaninglessByte = 0;
+
while (hasMore) {
results.clear();
hasMore = innerScanner.nextRaw(results);
@@ -76,21 +79,34 @@ public class AggregationScanner implements RegionScanner {
Cell cell = results.get(0);
tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER.ordinal()) {
- if (filter != null && filter.evaluate(tuple) == false)
- continue;
-
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR.ordinal()) {
- AggrKey aggKey = projector.getAggrKey(results);
- MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
- aggregators.aggregate(bufs, results);
-
- if (behavior.ordinal() >= ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
- aggCache.checkMemoryUsage();
+ if (behavior == CoprocessorBehavior.SCAN) {
+ //touch every byte of the cell so that the cost of scanning will be trully reflected
+ int endIndex = cell.getRowOffset() + cell.getRowLength();
+ for (int i = cell.getRowOffset(); i < endIndex; ++i) {
+ meaninglessByte += cell.getRowArray()[i];
+ }
+ } else {
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal()) {
+ if (filter != null && filter.evaluate(tuple) == false)
+ continue;
+
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal()) {
+ AggrKey aggKey = projector.getAggrKey(results);
+ MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
+ aggregators.aggregate(bufs, results);
+
+ if (behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
+ aggCache.checkMemoryUsage();
+ }
}
}
}
}
+
+ if (behavior == CoprocessorBehavior.SCAN) {
+ System.out.println("meaningless byte is now " + meaninglessByte);
+ }
+
return aggCache;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
deleted file mode 100644
index 7630d57..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverBehavior.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-/**
- */
-public enum ObserverBehavior {
- SCAN, //only scan data, used for profiling tuple scan speed
- SCAN_FILTER, //only scan+filter used,used for profiling filter speed
- SCAN_FILTER_AGGR, //aggregate the result
- SCAN_FILTER_AGGR_CHECKMEM, //default full operations
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index 7db2416..4750ea4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -35,6 +35,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
@@ -74,14 +75,14 @@ public class ObserverEnabler {
if (localCoprocessor) {
RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
return new ResultScannerAdapter(aggrScanner);
} else {
// debug/profiling purpose
- String toggle = BackdoorToggles.getObserverBehavior();
+ String toggle = BackdoorToggles.getCoprocessorBehavior();
if (toggle == null) {
- toggle = ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
} else {
logger.info("The execution of this query will use " + toggle + " as observer's behavior");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5674986..77a5db5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.DataFormatException;
import javax.annotation.Nullable;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -47,6 +49,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
@@ -64,11 +67,13 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private GTInfo info;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
+ private int totalScannedCount;
- public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns) {
+ public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, int totalScannedCount) {
this.info = info;
this.blocks = blocks;
this.columns = columns;
+ this.totalScannedCount = totalScannedCount;
}
@Override
@@ -78,7 +83,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
@Override
public int getScannedRowCount() {
- return 0;
+ return totalScannedCount;
}
@Override
@@ -155,6 +160,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logScan(rawScan, cubeSeg.getStorageLocationIdentifier());
}
+ final AtomicInteger totalScannedCount = new AtomicInteger(0);
+
for (int i = 0; i < rawScans.size(); ++i) {
final int shardIndex = i;
final RawScan rawScan = rawScans.get(i);
@@ -169,6 +176,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
+ // debug/profiling purpose
+ String toggle = BackdoorToggles.getCoprocessorBehavior();
+ if (toggle == null) {
+ toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
+ } else {
+ logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
+ }
+ builder.setBehavior(toggle);
+
Collection<CubeVisitProtos.CubeVisitResponse> results;
try {
results = getResults(builder.build(), hbaseTable, rawScan.startKey, rawScan.endKey);
@@ -182,6 +198,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
for (CubeVisitProtos.CubeVisitResponse result : results) {
+ totalScannedCount.addAndGet(result.getStats().getScannedRowCount());
logger.info(getStatsString(result, shardIndex));
}
@@ -209,7 +226,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
throw new RuntimeException("Visiting cube by endpoint gets interrupted");
}
- return new EndpointResultsAsGTScanner(fullGTInfo, rowBlocks.iterator(), scanRequest.getColumns());
+ return new EndpointResultsAsGTScanner(fullGTInfo, rowBlocks.iterator(), scanRequest.getColumns(), totalScannedCount.get());
}
private String getStatsString(CubeVisitProtos.CubeVisitResponse result, int shardIndex) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index fb497b9..f474139 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -41,6 +41,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.CellListIterator;
import org.apache.kylin.storage.hbase.cube.v2.CubeHBaseRPC;
import org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore;
@@ -56,7 +57,8 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
-@SuppressWarnings("unused")//used in hbase endpoint
+@SuppressWarnings("unused")
+//used in hbase endpoint
public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);
@@ -144,7 +146,12 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScan.hbaseColumns, hbaseColumnsToGT);
IGTScanner rawScanner = store.scan(scanReq);
- IGTScanner finalScanner = scanReq.decorateScanner(rawScanner);
+
+ CoprocessorBehavior behavior = CoprocessorBehavior.valueOf(request.getBehavior());
+ IGTScanner finalScanner = scanReq.decorateScanner(rawScanner,//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER.ordinal(),//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR.ordinal(),//
+ behavior.ordinal() >= CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal());//
ByteBuffer buffer = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 225703d..0923484 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -11,47 +11,62 @@ public final class CubeVisitProtos {
public interface CubeVisitRequestOrBuilder
extends com.google.protobuf.MessageOrBuilder {
- // required bytes gtScanRequest = 1;
+ // required string behavior = 1;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required string behavior = 1;</code>
+ */
+ boolean hasBehavior();
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ java.lang.String getBehavior();
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getBehaviorBytes();
+
+ // required bytes gtScanRequest = 2;
+ /**
+ * <code>required bytes gtScanRequest = 2;</code>
*/
boolean hasGtScanRequest();
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
com.google.protobuf.ByteString getGtScanRequest();
- // required bytes hbaseRawScan = 2;
+ // required bytes hbaseRawScan = 3;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
boolean hasHbaseRawScan();
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
com.google.protobuf.ByteString getHbaseRawScan();
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>
getHbaseColumnsToGTList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index);
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
int getHbaseColumnsToGTCount();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList();
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index);
@@ -109,18 +124,23 @@ public final class CubeVisitProtos {
}
case 10: {
bitField0_ |= 0x00000001;
- gtScanRequest_ = input.readBytes();
+ behavior_ = input.readBytes();
break;
}
case 18: {
bitField0_ |= 0x00000002;
- hbaseRawScan_ = input.readBytes();
+ gtScanRequest_ = input.readBytes();
break;
}
case 26: {
- if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ bitField0_ |= 0x00000004;
+ hbaseRawScan_ = input.readBytes();
+ break;
+ }
+ case 34: {
+ if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>();
- mutable_bitField0_ |= 0x00000004;
+ mutable_bitField0_ |= 0x00000008;
}
hbaseColumnsToGT_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.PARSER, extensionRegistry));
break;
@@ -133,7 +153,7 @@ public final class CubeVisitProtos {
throw new com.google.protobuf.InvalidProtocolBufferException(
e.getMessage()).setUnfinishedMessage(this);
} finally {
- if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) {
+ if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
}
this.unknownFields = unknownFields.build();
@@ -672,68 +692,111 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required bytes gtScanRequest = 1;
- public static final int GTSCANREQUEST_FIELD_NUMBER = 1;
+ // required string behavior = 1;
+ public static final int BEHAVIOR_FIELD_NUMBER = 1;
+ private java.lang.Object behavior_;
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public boolean hasBehavior() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public java.lang.String getBehavior() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ behavior_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getBehaviorBytes() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ behavior_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required bytes gtScanRequest = 2;
+ public static final int GTSCANREQUEST_FIELD_NUMBER = 2;
private com.google.protobuf.ByteString gtScanRequest_;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
- // required bytes hbaseRawScan = 2;
- public static final int HBASERAWSCAN_FIELD_NUMBER = 2;
+ // required bytes hbaseRawScan = 3;
+ public static final int HBASERAWSCAN_FIELD_NUMBER = 3;
private com.google.protobuf.ByteString hbaseRawScan_;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
- public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
+ public static final int HBASECOLUMNSTOGT_FIELD_NUMBER = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
return hbaseColumnsToGT_;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
return hbaseColumnsToGT_.size();
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
return hbaseColumnsToGT_.get(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -741,6 +804,7 @@ public final class CubeVisitProtos {
}
private void initFields() {
+ behavior_ = "";
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
hbaseColumnsToGT_ = java.util.Collections.emptyList();
@@ -750,6 +814,10 @@ public final class CubeVisitProtos {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
+ if (!hasBehavior()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
if (!hasGtScanRequest()) {
memoizedIsInitialized = 0;
return false;
@@ -766,13 +834,16 @@ public final class CubeVisitProtos {
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, gtScanRequest_);
+ output.writeBytes(1, getBehaviorBytes());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, hbaseRawScan_);
+ output.writeBytes(2, gtScanRequest_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeBytes(3, hbaseRawScan_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
- output.writeMessage(3, hbaseColumnsToGT_.get(i));
+ output.writeMessage(4, hbaseColumnsToGT_.get(i));
}
getUnknownFields().writeTo(output);
}
@@ -785,15 +856,19 @@ public final class CubeVisitProtos {
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, gtScanRequest_);
+ .computeBytesSize(1, getBehaviorBytes());
}
if (((bitField0_ & 0x00000002) == 0x00000002)) {
size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, hbaseRawScan_);
+ .computeBytesSize(2, gtScanRequest_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, hbaseRawScan_);
}
for (int i = 0; i < hbaseColumnsToGT_.size(); i++) {
size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(3, hbaseColumnsToGT_.get(i));
+ .computeMessageSize(4, hbaseColumnsToGT_.get(i));
}
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
@@ -818,6 +893,11 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other = (org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest) obj;
boolean result = true;
+ result = result && (hasBehavior() == other.hasBehavior());
+ if (hasBehavior()) {
+ result = result && getBehavior()
+ .equals(other.getBehavior());
+ }
result = result && (hasGtScanRequest() == other.hasGtScanRequest());
if (hasGtScanRequest()) {
result = result && getGtScanRequest()
@@ -843,6 +923,10 @@ public final class CubeVisitProtos {
}
int hash = 41;
hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasBehavior()) {
+ hash = (37 * hash) + BEHAVIOR_FIELD_NUMBER;
+ hash = (53 * hash) + getBehavior().hashCode();
+ }
if (hasGtScanRequest()) {
hash = (37 * hash) + GTSCANREQUEST_FIELD_NUMBER;
hash = (53 * hash) + getGtScanRequest().hashCode();
@@ -965,13 +1049,15 @@ public final class CubeVisitProtos {
public Builder clear() {
super.clear();
- gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
+ behavior_ = "";
bitField0_ = (bitField0_ & ~0x00000001);
- hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
+ gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
bitField0_ = (bitField0_ & ~0x00000002);
+ hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
+ bitField0_ = (bitField0_ & ~0x00000004);
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
hbaseColumnsToGTBuilder_.clear();
}
@@ -1006,15 +1092,19 @@ public final class CubeVisitProtos {
if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
to_bitField0_ |= 0x00000001;
}
- result.gtScanRequest_ = gtScanRequest_;
+ result.behavior_ = behavior_;
if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
to_bitField0_ |= 0x00000002;
}
+ result.gtScanRequest_ = gtScanRequest_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
result.hbaseRawScan_ = hbaseRawScan_;
if (hbaseColumnsToGTBuilder_ == null) {
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = java.util.Collections.unmodifiableList(hbaseColumnsToGT_);
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
}
result.hbaseColumnsToGT_ = hbaseColumnsToGT_;
} else {
@@ -1036,6 +1126,11 @@ public final class CubeVisitProtos {
public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest other) {
if (other == org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.getDefaultInstance()) return this;
+ if (other.hasBehavior()) {
+ bitField0_ |= 0x00000001;
+ behavior_ = other.behavior_;
+ onChanged();
+ }
if (other.hasGtScanRequest()) {
setGtScanRequest(other.getGtScanRequest());
}
@@ -1046,7 +1141,7 @@ public final class CubeVisitProtos {
if (!other.hbaseColumnsToGT_.isEmpty()) {
if (hbaseColumnsToGT_.isEmpty()) {
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
} else {
ensureHbaseColumnsToGTIsMutable();
hbaseColumnsToGT_.addAll(other.hbaseColumnsToGT_);
@@ -1059,7 +1154,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_.dispose();
hbaseColumnsToGTBuilder_ = null;
hbaseColumnsToGT_ = other.hbaseColumnsToGT_;
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
hbaseColumnsToGTBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getHbaseColumnsToGTFieldBuilder() : null;
@@ -1073,6 +1168,10 @@ public final class CubeVisitProtos {
}
public final boolean isInitialized() {
+ if (!hasBehavior()) {
+
+ return false;
+ }
if (!hasGtScanRequest()) {
return false;
@@ -1103,85 +1202,159 @@ public final class CubeVisitProtos {
}
private int bitField0_;
- // required bytes gtScanRequest = 1;
+ // required string behavior = 1;
+ private java.lang.Object behavior_ = "";
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public boolean hasBehavior() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public java.lang.String getBehavior() {
+ java.lang.Object ref = behavior_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ behavior_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getBehaviorBytes() {
+ java.lang.Object ref = behavior_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ behavior_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder setBehavior(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ behavior_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder clearBehavior() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ behavior_ = getDefaultInstance().getBehavior();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string behavior = 1;</code>
+ */
+ public Builder setBehaviorBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ behavior_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required bytes gtScanRequest = 2;
private com.google.protobuf.ByteString gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public boolean hasGtScanRequest() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
+ return ((bitField0_ & 0x00000002) == 0x00000002);
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public com.google.protobuf.ByteString getGtScanRequest() {
return gtScanRequest_;
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public Builder setGtScanRequest(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000001;
+ bitField0_ |= 0x00000002;
gtScanRequest_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes gtScanRequest = 1;</code>
+ * <code>required bytes gtScanRequest = 2;</code>
*/
public Builder clearGtScanRequest() {
- bitField0_ = (bitField0_ & ~0x00000001);
+ bitField0_ = (bitField0_ & ~0x00000002);
gtScanRequest_ = getDefaultInstance().getGtScanRequest();
onChanged();
return this;
}
- // required bytes hbaseRawScan = 2;
+ // required bytes hbaseRawScan = 3;
private com.google.protobuf.ByteString hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public boolean hasHbaseRawScan() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
+ return ((bitField0_ & 0x00000004) == 0x00000004);
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public com.google.protobuf.ByteString getHbaseRawScan() {
return hbaseRawScan_;
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public Builder setHbaseRawScan(com.google.protobuf.ByteString value) {
if (value == null) {
throw new NullPointerException();
}
- bitField0_ |= 0x00000002;
+ bitField0_ |= 0x00000004;
hbaseRawScan_ = value;
onChanged();
return this;
}
/**
- * <code>required bytes hbaseRawScan = 2;</code>
+ * <code>required bytes hbaseRawScan = 3;</code>
*/
public Builder clearHbaseRawScan() {
- bitField0_ = (bitField0_ & ~0x00000002);
+ bitField0_ = (bitField0_ & ~0x00000004);
hbaseRawScan_ = getDefaultInstance().getHbaseRawScan();
onChanged();
return this;
}
- // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;
+ // repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;
private java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> hbaseColumnsToGT_ =
java.util.Collections.emptyList();
private void ensureHbaseColumnsToGTIsMutable() {
- if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+ if (!((bitField0_ & 0x00000008) == 0x00000008)) {
hbaseColumnsToGT_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList>(hbaseColumnsToGT_);
- bitField0_ |= 0x00000004;
+ bitField0_ |= 0x00000008;
}
}
@@ -1189,7 +1362,7 @@ public final class CubeVisitProtos {
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder> hbaseColumnsToGTBuilder_;
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> getHbaseColumnsToGTList() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1199,7 +1372,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public int getHbaseColumnsToGTCount() {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1209,7 +1382,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList getHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1219,7 +1392,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1236,7 +1409,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder setHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1250,7 +1423,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1266,7 +1439,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList value) {
@@ -1283,7 +1456,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1297,7 +1470,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addHbaseColumnsToGT(
int index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder builderForValue) {
@@ -1311,7 +1484,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder addAllHbaseColumnsToGT(
java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList> values) {
@@ -1325,12 +1498,12 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder clearHbaseColumnsToGT() {
if (hbaseColumnsToGTBuilder_ == null) {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000004);
+ bitField0_ = (bitField0_ & ~0x00000008);
onChanged();
} else {
hbaseColumnsToGTBuilder_.clear();
@@ -1338,7 +1511,7 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public Builder removeHbaseColumnsToGT(int index) {
if (hbaseColumnsToGTBuilder_ == null) {
@@ -1351,14 +1524,14 @@ public final class CubeVisitProtos {
return this;
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder getHbaseColumnsToGTBuilder(
int index) {
return getHbaseColumnsToGTFieldBuilder().getBuilder(index);
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder getHbaseColumnsToGTOrBuilder(
int index) {
@@ -1368,7 +1541,7 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>
getHbaseColumnsToGTOrBuilderList() {
@@ -1379,14 +1552,14 @@ public final class CubeVisitProtos {
}
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder() {
return getHbaseColumnsToGTFieldBuilder().addBuilder(
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder addHbaseColumnsToGTBuilder(
int index) {
@@ -1394,7 +1567,7 @@ public final class CubeVisitProtos {
index, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.getDefaultInstance());
}
/**
- * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 3;</code>
+ * <code>repeated .CubeVisitRequest.IntList hbaseColumnsToGT = 4;</code>
*/
public java.util.List<org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder>
getHbaseColumnsToGTBuilderList() {
@@ -1407,7 +1580,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGTBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList.Builder, org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntListOrBuilder>(
hbaseColumnsToGT_,
- ((bitField0_ & 0x00000004) == 0x00000004),
+ ((bitField0_ & 0x00000008) == 0x00000008),
getParentForChildren(),
isClean());
hbaseColumnsToGT_ = null;
@@ -1606,10 +1779,6 @@ public final class CubeVisitProtos {
}
/**
* Protobuf type {@code CubeVisitResponse.Stats}
- *
- * <pre>
- *all entries in this struct be optional to conveniently add more entries in the future
- * </pre>
*/
public static final class Stats extends
com.google.protobuf.GeneratedMessage
@@ -1983,10 +2152,6 @@ public final class CubeVisitProtos {
}
/**
* Protobuf type {@code CubeVisitResponse.Stats}
- *
- * <pre>
- *all entries in this struct be optional to conveniently add more entries in the future
- * </pre>
*/
public static final class Builder extends
com.google.protobuf.GeneratedMessage.Builder<Builder>
@@ -3055,20 +3220,21 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\215\001\n\020C" +
- "ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
- "\n\014hbaseRawScan\030\002 \002(\014\0223\n\020hbaseColumnsToGT" +
- "\030\003 \003(\0132\031.CubeVisitRequest.IntList\032\027\n\007Int" +
- "List\022\014\n\004ints\030\001 \003(\005\"\304\001\n\021CubeVisitResponse" +
- "\022\026\n\016compressedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132" +
- "\030.CubeVisitResponse.Stats\032n\n\005Stats\022\030\n\020se" +
- "rviceStartTime\030\001 \001(\003\022\026\n\016serviceEndTime\030\002",
- " \001(\003\022\027\n\017scannedRowCount\030\003 \001(\005\022\032\n\022aggrega" +
- "tedRowCount\030\004 \001(\0052F\n\020CubeVisitService\0222\n" +
- "\tvisitCube\022\021.CubeVisitRequest\032\022.CubeVisi" +
- "tResponseB`\nEorg.apache.kylin.storage.hb" +
- "ase.cube.v2.coprocessor.endpoint.generat" +
- "edB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "/endpoint/protobuf/CubeVisit.proto\"\237\001\n\020C" +
+ "ubeVisitRequest\022\020\n\010behavior\030\001 \002(\t\022\025\n\rgtS" +
+ "canRequest\030\002 \002(\014\022\024\n\014hbaseRawScan\030\003 \002(\014\0223" +
+ "\n\020hbaseColumnsToGT\030\004 \003(\0132\031.CubeVisitRequ" +
+ "est.IntList\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\304\001\n" +
+ "\021CubeVisitResponse\022\026\n\016compressedRows\030\001 \002" +
+ "(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitResponse.St" +
+ "ats\032n\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022\026",
+ "\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCoun" +
+ "t\030\003 \001(\005\022\032\n\022aggregatedRowCount\030\004 \001(\0052F\n\020C" +
+ "ubeVisitService\0222\n\tvisitCube\022\021.CubeVisit" +
+ "Request\032\022.CubeVisitResponseB`\nEorg.apach" +
+ "e.kylin.storage.hbase.cube.v2.coprocesso" +
+ "r.endpoint.generatedB\017CubeVisitProtosH\001\210" +
+ "\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -3080,7 +3246,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "HbaseColumnsToGT", });
+ new java.lang.String[] { "Behavior", "GtScanRequest", "HbaseRawScan", "HbaseColumnsToGT", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index a4cd39d..4ac6414 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -12,16 +12,16 @@ option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
message CubeVisitRequest {
- required bytes gtScanRequest = 1;
- required bytes hbaseRawScan = 2;
- repeated IntList hbaseColumnsToGT = 3;
+ required string behavior = 1;
+ required bytes gtScanRequest = 2;
+ required bytes hbaseRawScan = 3;
+ repeated IntList hbaseColumnsToGT = 4;
message IntList {
repeated int32 ints = 1;
}
}
message CubeVisitResponse {
- //all entries in this struct be optional to conveniently add more entries in the future
message Stats {
optional int64 serviceStartTime = 1;
optional int64 serviceEndTime = 2;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c37bc653/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 95a9128..a88e147 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -41,6 +41,7 @@ import org.apache.kylin.metadata.measure.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
@@ -123,7 +124,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
@@ -172,7 +173,7 @@ public class AggregateRegionObserverTest {
MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, ObserverBehavior.SCAN_FILTER_AGGR_CHECKMEM);
+ RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM);
ArrayList<Cell> result = Lists.newArrayList();
boolean hasMore = true;
while (hasMore) {
[2/2] incubator-kylin git commit: KYLIN-942 bug fixs and minor
changes on configurations
Posted by ma...@apache.org.
KYLIN-942 bug fixs and minor changes on configurations
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d2e2d21b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d2e2d21b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d2e2d21b
Branch: refs/heads/2.x-staging
Commit: d2e2d21bcfd8e1c5e961af15fbbeac9a742c1e48
Parents: 480bc0b
Author: honma <ho...@ebay.com>
Authored: Fri Oct 30 15:34:20 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 2 15:52:56 2015 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 6 +++---
core-common/pom.xml | 6 ------
dev-support/test_all.sh | 11 -----------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 11 +++++++++++
.../org/apache/kylin/query/test/ITKylinQueryTest.java | 2 +-
query/src/test/resources/query/temp/.gitignore | 1 +
.../cube/v2/coprocessor/endpoint/CubeVisitService.java | 1 +
7 files changed, 17 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index fe953fc..cc91824 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -56,9 +56,9 @@ kylin.job.cubing.inMem.sampling.percent=100
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
-kylin.hbase.region.cut.small=10
-kylin.hbase.region.cut.medium=20
-kylin.hbase.region.cut.large=100
+kylin.hbase.region.cut.small=5
+kylin.hbase.region.cut.medium=10
+kylin.hbase.region.cut.large=50
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index f7444be..3275532 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -89,7 +89,6 @@
<groupId>com.ning</groupId>
<artifactId>compress-lzf</artifactId>
</dependency>
-
<!-- Logging -->
<dependency>
@@ -115,11 +114,6 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/dev-support/test_all.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all.sh b/dev-support/test_all.sh
deleted file mode 100644
index 6a7b887..0000000
--- a/dev-support/test_all.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests | tee mci.log
-mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithEngineTest.log
-mvn test -Dtest=org.apache.kylin.job.BuildIIWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildIIWithStreamTest.log
-mvn test -Dtest=org.apache.kylin.job.BuildCubeWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithStreamTest.log
-mvn verify -fae -P sandbox | tee mvnverify.log
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100644
index 0000000..1f63f71
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests | tee mci.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithEngineTest.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildIIWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildIIWithStreamTest.log
+mvn test -Dhdp.version=2.2.4.2-2 -Dtest=org.apache.kylin.job.BuildCubeWithStreamTest -DfailIfNoTests=false -P sandbox | tee BuildCubeWithStreamTest.log
+mvn verify -Dhdp.version=2.2.4.2-2 -fae -P sandbox | tee mvnverify.log
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
index 4821ce9..72c366b 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -95,7 +95,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql/query82.sql";
+ String queryFileName = "src/test/resources/query/temp/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/query/src/test/resources/query/temp/.gitignore
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/temp/.gitignore b/query/src/test/resources/query/temp/.gitignore
new file mode 100644
index 0000000..d1b811b
--- /dev/null
+++ b/query/src/test/resources/query/temp/.gitignore
@@ -0,0 +1 @@
+*.sql
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d2e2d21b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index ba766bd..fb497b9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -56,6 +56,7 @@ import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
+@SuppressWarnings("unused")//used in hbase endpoint
public class CubeVisitService extends CubeVisitProtos.CubeVisitService implements Coprocessor, CoprocessorService {
private static final Logger logger = LoggerFactory.getLogger(CubeVisitService.class);