You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/26 03:34:12 UTC
[53/55] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d9a59be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d9a59be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d9a59be
Branch: refs/heads/1.5.x-CDH5.7
Commit: 7d9a59be2cba4fb0cb08f8a2ee28b7217ea54f52
Parents: 5d679f1
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jul 26 09:49:00 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../kylin/provision/BuildCubeWithEngine.java | 2 ++
.../org/apache/kylin/rest/service/CubeService.java | 2 ++
.../org/apache/kylin/rest/service/UserService.java | 1 -
.../kylin/storage/hbase/HBaseConnection.java | 1 -
.../kylin/storage/hbase/HBaseResourceStore.java | 12 +++++++-----
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 17 +++++++++++++++++
.../storage/hbase/steps/HBaseCuboidWriter.java | 16 ++++++++--------
.../storage/hbase/util/StorageCleanupJob.java | 1 -
9 files changed, 40 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b146031..87b7c22 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -781,4 +781,8 @@ abstract public class KylinConfigBase implements Serializable {
public void setAppendDictCacheSize(int cacheSize) {
setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize));
}
+
+ public String getPatchedFuzzyRowFilterVersion() {
+ return this.getOptional("kylin.hbase.filter.fuzzy.row.filter.version", "1.1.3");
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 2a5979f..30e95d3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -58,6 +58,8 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.model.IEngineAware;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 0503236..4e5d793 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.HTable;
import java.util.*;
+import java.util.*;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index e039534..0e0e213 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -152,7 +152,6 @@ public class UserService implements UserDetailsManager {
Put put = new Put(pair.getKey());
put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
-
htable.put(put);
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 7dbc7f0..e7ee2f5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -283,4 +283,3 @@ public class HBaseConnection {
}
}
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index aa7a4d4..9881569 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -33,8 +33,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -324,9 +322,13 @@ public class HBaseResourceStore extends ResourceStore {
get.addColumn(B_FAMILY, B_COLUMN_TS);
}
- Result result = table.get(get);
- boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
- return exists ? result : null;
+ try {
+ Result result = table.get(get);
+ boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
+ return exists ? result : null;
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
}
private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d84074f..9f787ad 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -435,4 +435,21 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
return sb.toString();
}
+
+ private Map<byte[], CubeVisitProtos.CubeVisitResponse> getResults(final CubeVisitProtos.CubeVisitRequest request, Table table, byte[] startKey, byte[] endKey) throws Throwable {
+ Map<byte[], CubeVisitProtos.CubeVisitResponse> results = table.coprocessorService(CubeVisitProtos.CubeVisitService.class, startKey, endKey, new Batch.Call<CubeVisitProtos.CubeVisitService, CubeVisitProtos.CubeVisitResponse>() {
+ public CubeVisitProtos.CubeVisitResponse call(CubeVisitProtos.CubeVisitService rowsService) throws IOException {
+ ServerRpcController controller = new ServerRpcController();
+ BlockingRpcCallback<CubeVisitProtos.CubeVisitResponse> rpcCallback = new BlockingRpcCallback<>();
+ rowsService.visitCube(controller, request, rpcCallback);
+ CubeVisitProtos.CubeVisitResponse response = rpcCallback.get();
+ if (controller.failedOnException()) {
+ throw controller.getFailedOn();
+ }
+ return response;
+ }
+ });
+
+ return results;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index c990379..a4a43c9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -128,15 +128,15 @@ public class HBaseCuboidWriter implements ICuboidWriter {
@Override
public final void flush() throws IOException {
- if (!puts.isEmpty()) {
- long t = System.currentTimeMillis();
- if (hTable != null) {
- hTable.put(puts);
+
+ if (!puts.isEmpty()) {
+ long t = System.currentTimeMillis();
+ if (hTable != null) {
+ hTable.put(puts);
+ }
+ logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
+ puts.clear();
}
- logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
- puts.clear();
- }
- puts.clear();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d9a59be/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 874121d..06eafa9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;