You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/02 03:59:48 UTC
[3/3] kylin git commit: KYLIN-2233 Support HBase 1.0.2
KYLIN-2233 Support HBase 1.0.2
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b75598a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b75598a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b75598a
Branch: refs/heads/yang22-hbase102
Commit: 6b75598ab8327dad5ca3bf40bcaf510c9810aae6
Parents: d4afbb9
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jan 2 11:58:10 2017 +0800
----------------------------------------------------------------------
pom.xml | 2 +-
.../storage/hbase/cube/v1/RegionScannerAdapter.java | 13 +++----------
.../coprocessor/observer/AggregateRegionObserver.java | 4 ++--
.../v1/coprocessor/observer/AggregationScanner.java | 14 ++++----------
.../observer/ObserverAggregationCache.java | 10 ++--------
.../observer/AggregateRegionObserverTest.java | 10 ++--------
.../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 3 +--
7 files changed, 15 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6d3425e..ebc8c9d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
<hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.0.2</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ public boolean next(List<Cell> result, int limit) throws IOException {
return next(result);
}
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ public boolean nextRaw(List<Cell> result, int limit) throws IOException {
return next(result);
}
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
public long getMvccReadPoint() {
return Long.MAX_VALUE;
}
-
- @Override
- public int getBatch() {
- return -1;
- }
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- Region region = ctxt.getEnvironment().getRegion();
+ HRegion region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index d64f48f..a900ea1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
@@ -117,8 +116,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
- return outerScanner.next(result, scannerContext);
+ public boolean next(List<Cell> result, int limit) throws IOException {
+ return outerScanner.next(result, limit);
}
@Override
@@ -127,8 +126,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
- return outerScanner.nextRaw(result, scannerContext);
+ public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ return outerScanner.nextRaw(result, limit);
}
@Override
@@ -161,11 +160,6 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
- @Override
- public int getBatch() {
- return outerScanner.getBatch();
- }
-
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 331e34d..8404262 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -113,7 +112,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ public boolean next(List<Cell> result, int limit) throws IOException {
return next(result);
}
@@ -123,7 +122,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ public boolean nextRaw(List<Cell> result, int limit) throws IOException {
return next(result);
}
@@ -162,11 +161,6 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
-
- @Override
- public int getBatch() {
- return innerScanner.getBatch();
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 4695353..5ec0238 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -231,7 +230,7 @@ public class AggregateRegionObserverTest {
}
@Override
- public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ public boolean next(List<Cell> result, int limit) throws IOException {
return next(result);
}
@@ -300,11 +299,6 @@ public class AggregateRegionObserverTest {
return 0;
}
- @Override
- public int getBatch() {
- return 0;
- }
-
/*
* (non-Javadoc)
*
@@ -322,7 +316,7 @@ public class AggregateRegionObserverTest {
}
@Override
- public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ public boolean nextRaw(List<Cell> list, int limit) throws IOException {
return false;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b75598a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 04e2e8b..4e87093 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -225,7 +224,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- Region first = regions.get(0);
+ HRegion first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();