You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/02/21 13:13:00 UTC
[3/3] kylin git commit: refactor
refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7a318c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7a318c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7a318c2
Branch: refs/heads/mhb0220
Commit: a7a318c214a8b03d8a133b7ec035765fceb5f734
Parents: 661f016
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Feb 20 10:15:53 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Feb 21 20:55:36 2017 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/ConcurrentDiskStore.java | 7 +----
.../cube/inmemcubing/InMemCubeBuilder.java | 5 ---
.../kylin/cube/inmemcubing/MemDiskStore.java | 6 ----
.../apache/kylin/gridtable/EmptyGTScanner.java | 9 +-----
.../kylin/gridtable/GTAggregateScanner.java | 7 +----
.../apache/kylin/gridtable/GTFilterScanner.java | 5 ---
.../apache/kylin/gridtable/GTScanRequest.java | 9 +++---
.../org/apache/kylin/gridtable/IGTScanner.java | 4 ---
.../benchmark/SortedGTRecordGenerator.java | 6 ----
.../gridtable/memstore/GTSimpleMemStore.java | 5 ---
.../gridtable/AggregationCacheSpillTest.java | 12 +------
.../kylin/gridtable/SimpleGridTableTest.java | 9 +-----
.../storage/gtrecord/CubeSegmentScanner.java | 5 ---
.../kylin/storage/gtrecord/ScannerWorker.java | 15 +++------
.../gtrecord/StorageResponseGTScatter.java | 9 +-----
.../hbase/cube/HBaseScannerBenchmark.java | 3 --
.../storage/hbase/cube/SimpleHBaseStore.java | 5 ---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 ++++++-------
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 33 ++------------------
.../hbase/cube/v2/HBaseReadonlyStore.java | 5 ---
.../endpoint/protobuf/CubeVisit.proto | 2 +-
.../hbase/steps/SandboxMetastoreCLI.java | 1 -
22 files changed, 27 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
index 5219ede..41d2dfb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/ConcurrentDiskStore.java
@@ -265,11 +265,6 @@ public class ConcurrentDiskStore implements IGTStore, Closeable {
return info;
}
- @Override
- public long getScannedRowCount() {
- return count;
- }
-
}
private class Writer implements IGTWriter {
@@ -371,4 +366,4 @@ public class ConcurrentDiskStore implements IGTStore, Closeable {
return "ConcurrentDiskStore@" + (info.getTableName() == null ? this.hashCode() : info.getTableName());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a74f0c0..e08844e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -564,10 +564,5 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
public GTInfo getInfo() {
return info;
}
-
- @Override
- public long getScannedRowCount() {
- return 0L;
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
index 81403ab..a5471df 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
@@ -277,12 +277,6 @@ public class MemDiskStore implements IGTStore, Closeable {
public GTInfo getInfo() {
return info;
}
-
- @Override
- public long getScannedRowCount() {
- return count;
- }
-
}
private class Writer implements IGTWriter {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
index 01d31f0..8b6c995 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/EmptyGTScanner.java
@@ -22,10 +22,8 @@ import java.io.IOException;
import java.util.Iterator;
public class EmptyGTScanner implements IGTScanner {
- private long reportScannedRowCount;
- public EmptyGTScanner(long reportScannedRowCount) {
- this.reportScannedRowCount = reportScannedRowCount;
+ public EmptyGTScanner() {
}
@Override
@@ -34,11 +32,6 @@ public class EmptyGTScanner implements IGTScanner {
}
@Override
- public long getScannedRowCount() {
- return reportScannedRowCount;
- }
-
- @Override
public void close() throws IOException {
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 8b0efcc..7cdd4f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -129,11 +129,6 @@ public class GTAggregateScanner implements IGTScanner {
}
@Override
- public long getScannedRowCount() {
- return inputScanner.getScannedRowCount();
- }
-
- @Override
public void close() throws IOException {
inputScanner.close();
aggrCache.close();
@@ -598,4 +593,4 @@ public class GTAggregateScanner implements IGTScanner {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index f1f84af..717f89c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -63,11 +63,6 @@ public class GTFilterScanner implements IGTScanner {
}
@Override
- public long getScannedRowCount() {
- return inputScanner.getScannedRowCount();
- }
-
- @Override
public void close() throws IOException {
inputScanner.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 651e5c4..4629c8e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -42,7 +42,7 @@ import com.google.common.collect.Sets;
public class GTScanRequest {
private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
-
+
//it's not necessary to increase the checkInterval to very large because the check cost is not high
//changing it might break org.apache.kylin.query.ITKylinQueryTest.testTimeoutQuery()
public static final int terminateCheckInterval = 100;
@@ -175,8 +175,8 @@ public class GTScanRequest {
public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, boolean spillEnabled) throws IOException {
IGTScanner result = scanner;
if (!filterToggledOn) { //Skip reading this section if you're not profiling!
- int scanned = lookAndForget(result);
- return new EmptyGTScanner(scanned);
+ lookAndForget(result);
+ return new EmptyGTScanner();
} else {
if (this.hasFilterPushDown() && !hasPreFiltered) {
@@ -184,9 +184,8 @@ public class GTScanRequest {
}
if (!aggrToggledOn) {//Skip reading this section if you're not profiling!
- long scanned = result.getScannedRowCount();
lookAndForget(result);
- return new EmptyGTScanner(scanned);
+ return new EmptyGTScanner();
}
if (!this.isAllowStorageAggregation()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
index 980787b..96c7972 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/IGTScanner.java
@@ -21,9 +21,5 @@ package org.apache.kylin.gridtable;
import java.io.Closeable;
public interface IGTScanner extends Iterable<GTRecord>, Closeable {
-
GTInfo getInfo();
-
- long getScannedRowCount();
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
index 9c839bb..71a0a21 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/SortedGTRecordGenerator.java
@@ -184,12 +184,6 @@ public class SortedGTRecordGenerator {
public GTInfo getInfo() {
return info;
}
-
- @Override
- public long getScannedRowCount() {
- return counter;
- }
-
}
private class Distribution {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
index f9f370b..e1b5406 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/memstore/GTSimpleMemStore.java
@@ -106,11 +106,6 @@ public class GTSimpleMemStore implements IGTStore {
}
@Override
- public long getScannedRowCount() {
- return count;
- }
-
- @Override
public void close() throws IOException {
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index 7abb069..8b2243c 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -68,11 +68,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
@Override
- public long getScannedRowCount() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void close() throws IOException {
}
@@ -109,11 +104,6 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
}
@Override
- public long getScannedRowCount() {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void close() throws IOException {
}
@@ -141,4 +131,4 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
assertEquals(10, count);
scanner.close();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index 4ac6644..14a25c5 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.gridtable;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -53,7 +52,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
GTBuilder builder = rebuild(table);
IGTScanner scanner = scan(table);
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
}
@Test
@@ -64,7 +62,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
GTBuilder builder = rebuild(table);
IGTScanner scanner = scan(table);
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
}
@Test
@@ -75,7 +72,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
GTBuilder builder = rebuild(table);
IGTScanner scanner = scanAndAggregate(table);
- assertEquals(builder.getWrittenRowCount(), scanner.getScannedRowCount());
}
@Test
@@ -86,7 +82,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
rebuildViaAppend(table);
IGTScanner scanner = scan(table);
- assertEquals(10, scanner.getScannedRowCount());
}
private IGTScanner scan(GridTable table) throws IOException {
@@ -101,12 +96,11 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
System.out.println(r);
}
scanner.close();
- System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
return scanner;
}
private IGTScanner scanAndAggregate(GridTable table) throws IOException {
- GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[]{"count", "sum"}).setFilterPushDown(null).createGTScanRequest();
+ GTScanRequest req = new GTScanRequestBuilder().setInfo(table.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(setOf(0, 2)).setAggrMetrics(setOf(3, 4)).setAggrMetricsFuncs(new String[] { "count", "sum" }).setFilterPushDown(null).createGTScanRequest();
IGTScanner scanner = table.scan(req);
int i = 0;
for (GTRecord r : scanner) {
@@ -135,7 +129,6 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
System.out.println(r);
}
scanner.close();
- System.out.println("Scanned Row Count: " + scanner.getScannedRowCount());
return scanner;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 029502c..4f206d4 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -96,11 +96,6 @@ public class CubeSegmentScanner implements IGTScanner {
return scanRequest == null ? null : scanRequest.getInfo();
}
- @Override
- public long getScannedRowCount() {
- return scanner.getScannedRowCount();
- }
-
public CubeSegment getSegment() {
return this.cubeSeg;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index fd50c54..9e89227 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -18,6 +18,10 @@
package org.apache.kylin.storage.gtrecord;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Iterator;
+
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.EmptyGTScanner;
import org.apache.kylin.gridtable.GTInfo;
@@ -29,10 +33,6 @@ import org.apache.kylin.metadata.model.ISegment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Iterator;
-
public class ScannerWorker {
private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
@@ -41,7 +41,7 @@ public class ScannerWorker {
public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
if (scanRequest == null) {
logger.info("Segment {} will be skipped", segment);
- internal = new EmptyGTScanner(0);
+ internal = new EmptyGTScanner();
return;
}
@@ -62,9 +62,4 @@ public class ScannerWorker {
public void close() throws IOException {
internal.close();
}
-
- public long getScannedRowCount() {
- return internal.getScannedRowCount();
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index dc8746f..3904b5c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -45,15 +45,13 @@ public class StorageResponseGTScatter implements IGTScanner {
private IPartitionStreamer partitionStreamer;
private Iterator<byte[]> blocks;
private ImmutableBitSet columns;
- private long totalScannedCount;
private int storagePushDownLimit = -1;
- public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, long totalScannedCount, int storagePushDownLimit) {
+ public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, int storagePushDownLimit) {
this.info = info;
this.partitionStreamer = partitionStreamer;
this.blocks = partitionStreamer.asByteArrayIterator();
this.columns = columns;
- this.totalScannedCount = totalScannedCount;
this.storagePushDownLimit = storagePushDownLimit;
}
@@ -63,11 +61,6 @@ public class StorageResponseGTScatter implements IGTScanner {
}
@Override
- public long getScannedRowCount() {
- return totalScannedCount;
- }
-
- @Override
public void close() throws IOException {
//If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource
partitionStreamer.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java
index 3fdb92f..3eecba1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/HBaseScannerBenchmark.java
@@ -132,9 +132,6 @@ public class HBaseScannerBenchmark {
}
scan.close();
- if (scan.getScannedRowCount() != count)
- throw new IllegalStateException();
-
t = System.currentTimeMillis() - t;
logger.info(msg + ", " + count + " records, " + speed(t) + "K rec/sec");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index f63d9c2..b12173d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -205,10 +205,5 @@ public class SimpleHBaseStore implements IGTStore {
public GTInfo getInfo() {
return info;
}
-
- @Override
- public long getScannedRowCount() {
- return count;
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index abc3437..82b67b6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -23,7 +23,6 @@ import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.DataFormatException;
@@ -146,8 +145,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size());
- final AtomicLong totalScannedCount = new AtomicLong(0);
-
// KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries
// for different cubes until redeployment of coprocessor jar.
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -205,7 +202,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
Stats stats = result.getStats();
queryContext.addAndGetScannedRows(stats.getScannedRowCount());
queryContext.addAndGetScannedBytes(stats.getScannedBytes());
- totalScannedCount.addAndGet(stats.getScannedRowCount());
// if any other region has responded with error, skip further processing
if (regionErrorHolder.get() != null) {
@@ -249,7 +245,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
});
}
- return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), totalScannedCount.get(), scanRequest.getStoragePushDownLimit());
+ return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), scanRequest.getStoragePushDownLimit());
}
private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
@@ -317,14 +313,14 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
CubeVisitResponse.ErrorInfo errorInfo = response.getErrorInfo();
switch (errorInfo.getType()) {
- case UNKNOWN_TYPE:
- return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage());
- case TIMEOUT:
- return new KylinTimeoutException(errorInfo.getMessage());
- case RESOURCE_LIMIT_EXCEEDED:
- return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage());
- default:
- throw new AssertionError("Unknown error type: " + errorInfo.getType());
+ case UNKNOWN_TYPE:
+ return new RuntimeException("Coprocessor aborts: " + errorInfo.getMessage());
+ case TIMEOUT:
+ return new KylinTimeoutException(errorInfo.getMessage());
+ case RESOURCE_LIMIT_EXCEEDED:
+ return new ResourceLimitExceededException("Coprocessor resource limit exceeded: " + errorInfo.getMessage());
+ default:
+ throw new AssertionError("Unknown error type: " + errorInfo.getType());
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 1698180..33f8d90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -93,31 +93,7 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
@Override
public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException {
- final IGTScanner scanner = getGTScannerInternal(scanRequest);
-
- return new IGTScanner() {
- @Override
- public GTInfo getInfo() {
- return scanner.getInfo();
- }
-
- @Override
- public long getScannedRowCount() {
- long sum = 0;
- sum += scanner.getScannedRowCount();
- return sum;
- }
-
- @Override
- public void close() throws IOException {
- scanner.close();
- }
-
- @Override
- public Iterator<GTRecord> iterator() {
- return scanner.iterator();
- }
- };
+ return getGTScannerInternal(scanRequest);
}
//for non-sharding cases it will only return one byte[] with not shard at beginning
@@ -229,11 +205,6 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
@Override
- public long getScannedRowCount() {
- return decorateScanner.getScannedRowCount();
- }
-
- @Override
public void close() throws IOException {
decorateScanner.close();
}
@@ -244,4 +215,4 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
index 4c02dff..631e8e8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/HBaseReadonlyStore.java
@@ -152,11 +152,6 @@ public class HBaseReadonlyStore implements IGTStore {
public GTInfo getInfo() {
return info;
}
-
- @Override
- public long getScannedRowCount() {
- return count;
- }
};
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index e01ff52..aa83595 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -37,7 +37,7 @@ message CubeVisitRequest {
required string kylinProperties = 5; // kylin properties
optional string queryId = 6;
optional bool spillEnabled = 7 [default = true];
- optional int64 maxScanBytes = 8; // 0 means no limit
+ optional int64 maxScanBytes = 8; // must be positive
message IntList {
repeated int32 ints = 1;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7a318c2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index 691886b..62b154e 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory;
* It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
*/
public class SandboxMetastoreCLI {
-
private static final Logger logger = LoggerFactory.getLogger(SandboxMetastoreCLI.class);
public static void main(String[] args) throws Exception {