You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/02 09:03:07 UTC
[4/4] kylin git commit: KYLIN-1924 Region server metrics: replace int
type for long type for scanned row count
KYLIN-1924 Region server metrics: replace int type for long type for scanned row count
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fd7eb97
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fd7eb97
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fd7eb97
Branch: refs/heads/master
Commit: 4fd7eb97a4953286fd2e4c95842135f21a48d13d
Parents: 165dca8
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jul 27 11:09:26 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 2 16:51:07 2016 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/ConcurrentDiskStore.java | 4 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 4 +-
.../kylin/cube/inmemcubing/MemDiskStore.java | 2 +-
.../apache/kylin/gridtable/EmptyGTScanner.java | 6 +-
.../kylin/gridtable/GTAggregateScanner.java | 9 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 2 +-
.../apache/kylin/gridtable/GTScanRequest.java | 14 +-
.../org/apache/kylin/gridtable/IGTScanner.java | 2 +-
.../apache/kylin/gridtable/ScannerWorker.java | 2 +-
.../benchmark/SortedGTRecordGenerator.java | 2 +-
.../gridtable/memstore/GTSimpleMemStore.java | 4 +-
.../gridtable/AggregationCacheSpillTest.java | 4 +-
.../apache/kylin/storage/StorageContext.java | 9 +
.../storage/gtrecord/CubeSegmentScanner.java | 4 +-
.../gtrecord/GTCubeStorageQueryBase.java | 12 +-
.../org/apache/kylin/engine/mr/HadoopUtil.java | 9 +-
.../org/apache/kylin/engine/mr/KylinMapper.java | 4 +
.../storage/hbase/cube/SimpleHBaseStore.java | 2 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 14 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 6 +-
.../hbase/cube/v2/HBaseReadonlyStore.java | 2 +-
.../endpoint/generated/CubeVisitProtos.java | 8629 +++++++++---------
.../endpoint/protobuf/CubeVisit.proto | 4 +-
23 files changed, 4455 insertions(+), 4295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
index d5d502c..2af8bce 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
@@ -154,7 +154,7 @@ public class ConcurrentDiskStore implements IGTStore, Closeable {
final DataInputStream din;
long fileLen;
long readOffset;
- int count;
+ long count;
Reader(long startOffset) throws IOException {
this.fileLen = diskFile.length();
@@ -265,7 +265,7 @@ public class ConcurrentDiskStore implements IGTStore, Closeable {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return count;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index fd9d03e..e12e815 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -558,8 +558,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
@Override
- public int getScannedRowCount() {
- return 0;
+ public long getScannedRowCount() {
+ return 0L;
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
index 40a15ee..d8cd814 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
@@ -278,7 +278,7 @@ public class MemDiskStore implements IGTStore, Closeable {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return count;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
index a6b6c47..01d31f0 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.util.Iterator;
public class EmptyGTScanner implements IGTScanner {
- private int reportScannedRowCount;
+ private long reportScannedRowCount;
- public EmptyGTScanner(int reportScannedRowCount) {
+ public EmptyGTScanner(long reportScannedRowCount) {
this.reportScannedRowCount = reportScannedRowCount;
}
@@ -34,7 +34,7 @@ public class EmptyGTScanner implements IGTScanner {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return reportScannedRowCount;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 103c8e5..cb23af4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -30,16 +30,16 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-import java.util.Map.Entry;
import java.util.PriorityQueue;
import java.util.SortedMap;
+import java.util.Map.Entry;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.MemoryBudgetController;
-import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
import org.apache.kylin.measure.BufferedMeasureEncoder;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureAggregators;
@@ -119,7 +119,7 @@ public class GTAggregateScanner implements IGTScanner {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return inputScanner.getScannedRowCount();
}
@@ -131,9 +131,12 @@ public class GTAggregateScanner implements IGTScanner {
@Override
public Iterator<GTRecord> iterator() {
+ long count = 0;
for (GTRecord r : inputScanner) {
+ count++;
aggrCache.aggregate(r);
}
+ logger.info("GTAggregateScanner input rows: " + count);
return aggrCache.iterator();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index a5fa808..31a9599 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -63,7 +63,7 @@ public class GTFilterScanner implements IGTScanner {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return inputScanner.getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index d4e6298..0ce6b4c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -33,12 +33,16 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.SerializeToByteBuffer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class GTScanRequest {
+ private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
+
private GTInfo info;
private List<GTScanRange> ranges;
private ImmutableBitSet columns;
@@ -160,16 +164,22 @@ public class GTScanRequest {
}
if (!doAggr) {//Skip reading this section if you're not profiling!
- int scanned = result.getScannedRowCount();
+ long scanned = result.getScannedRowCount();
lookAndForget(result);
return new EmptyGTScanner(scanned);
}
- if (this.allowPreAggregation && this.hasAggregation()) {
+ if (!this.allowPreAggregation) {
+ logger.info("pre aggregation is not beneficial, skip it");
+ } else if (this.hasAggregation()) {
+ logger.info("pre aggregating results before returning");
result = new GTAggregateScanner(result, this);
+ } else {
+ logger.info("has no aggregation, skip it");
}
return result;
}
+
}
//touch every byte of the cell so that the cost of scanning will be truly reflected
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
index c0da569..980787b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
@@ -24,6 +24,6 @@ public interface IGTScanner extends Iterable<GTRecord>, Closeable {
GTInfo getInfo();
- int getScannedRowCount();
+ long getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 9c3b391..5be91be 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -59,7 +59,7 @@ public class ScannerWorker {
internal.close();
}
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return internal.getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
index 094acec..9c839bb 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
@@ -186,7 +186,7 @@ public class SortedGTRecordGenerator {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return counter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index ec5d22d..f9f370b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -98,7 +98,7 @@ public class GTSimpleMemStore implements IGTStore {
public IGTScanner scan(GTScanRequest scanRequest) {
return new IGTScanner() {
- int count;
+ long count;
@Override
public GTInfo getInfo() {
@@ -106,7 +106,7 @@ public class GTSimpleMemStore implements IGTStore {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return count;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index c92e1aa..4160f86 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -70,7 +70,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
throw new UnsupportedOperationException();
}
@@ -112,7 +112,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 90f950c..90a2e43 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -39,6 +39,7 @@ public class StorageContext {
private boolean acceptPartialResult;
private boolean exactAggregation;
+ private boolean needStorageAggregation;
private boolean enableLimit;
private boolean enableCoprocessor;
@@ -143,6 +144,14 @@ public class StorageContext {
this.partialResultReturned = partialResultReturned;
}
+ public boolean isNeedStorageAggregation() {
+ return needStorageAggregation;
+ }
+
+ public void setNeedStorageAggregation(boolean needStorageAggregation) {
+ this.needStorageAggregation = needStorageAggregation;
+ }
+
public void setExactAggregation(boolean isExactAggregation) {
this.exactAggregation = isExactAggregation;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 15e3272..9ca53f9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,7 +69,7 @@ public class CubeSegmentScanner implements IGTScanner {
}
scanRequest = scanRangePlanner.planScanRequest();
if (scanRequest != null) {
- scanRequest.setAllowPreAggregation(!context.isExactAggregation());
+ scanRequest.setAllowPreAggregation(context.isNeedStorageAggregation());
scanRequest.setAggrCacheGB(cubeSeg.getCubeInstance().getConfig().getQueryCoprocessorMemGB());
if (context.isLimitEnabled())
scanRequest.setRowLimit(context.getLimit());
@@ -93,7 +93,7 @@ public class CubeSegmentScanner implements IGTScanner {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return scanner.getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 65aa90a..7acf186 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -99,8 +99,9 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// isExactAggregation? meaning: tuples returned from storage requires no further aggregation in query engine
Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
- boolean isExactAggregation = isExactAggregation(cuboid, groups, otherDimsD, singleValuesD, derivedPostAggregation);
- context.setExactAggregation(isExactAggregation);
+ boolean exactAggregation = isExactAggregation(cuboid, groups, otherDimsD, singleValuesD, derivedPostAggregation);
+ context.setExactAggregation(exactAggregation);
+ context.setNeedStorageAggregation(isNeedStorageAggregation(cuboid, groupsD, singleValuesD, exactAggregation));
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
TupleFilter filterD = translateDerived(filter, groupsD);
@@ -221,7 +222,12 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return resultD;
}
- private boolean isExactAggregation(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation) {
+ public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD, boolean isExactAggregation) {
+ logger.info("Set isNeedStorageAggregation to " + !isExactAggregation);
+ return !isExactAggregation;
+ }
+
+ public boolean isExactAggregation(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation) {
boolean exact = true;
if (cuboid.requirePostAggregation()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index c638d99..f5ed81d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -30,8 +30,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Writable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HadoopUtil {
+ private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
private static final ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
public static void setCurrentConfiguration(Configuration conf) {
@@ -41,9 +44,13 @@ public class HadoopUtil {
public static Configuration getCurrentConfiguration() {
if (hadoopConfig.get() == null) {
Configuration conf = healSickConfig(new Configuration());
+ hadoopConfig.set(conf);
+ logger.info("The conf for current mapper will be " + System.identityHashCode(conf));
return conf;
}
- return hadoopConfig.get();
+ Configuration conf = hadoopConfig.get();
+ logger.info("The conf for current mapper will be " + System.identityHashCode(conf));
+ return conf;
}
private static Configuration healSickConfig(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index 0c919f7..29c6844 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -20,12 +20,16 @@ package org.apache.kylin.engine.mr;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
+ private static final Logger logger = LoggerFactory.getLogger(KylinMapper.class);
protected void bindCurrentConfiguration(Configuration conf) {
+ logger.info("The conf for current mapper will be " + System.identityHashCode(conf));
HadoopUtil.setCurrentConfiguration(conf);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index 2462873..b141190 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -207,7 +207,7 @@ public class SimpleHBaseStore implements IGTStore {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return count;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 163226b..1cebdea 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -27,7 +27,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
import javax.annotation.Nullable;
@@ -60,10 +60,10 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -165,9 +165,9 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private GTInfo info;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
- private int totalScannedCount;
+ private long totalScannedCount;
- public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, int totalScannedCount) {
+ public EndpointResultsAsGTScanner(GTInfo info, Iterator<byte[]> blocks, ImmutableBitSet columns, long totalScannedCount) {
this.info = info;
this.blocks = blocks;
this.columns = columns;
@@ -180,7 +180,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return totalScannedCount;
}
@@ -330,7 +330,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size());
- final AtomicInteger totalScannedCount = new AtomicInteger(0);
+ final AtomicLong totalScannedCount = new AtomicLong(0);
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum);
// KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index dcedf76..238939c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -110,8 +110,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
@Override
- public int getScannedRowCount() {
- int sum = 0;
+ public long getScannedRowCount() {
+ long sum = 0;
sum += scanner.getScannedRowCount();
return sum;
}
@@ -226,7 +226,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return decorateScanner.getScannedRowCount();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fd7eb97/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 850b3d9..4b9b4fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -141,7 +141,7 @@ public class HBaseReadonlyStore implements IGTStore {
}
@Override
- public int getScannedRowCount() {
+ public long getScannedRowCount() {
return count;
}
};