You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/06/20 04:24:02 UTC
kylin git commit: KYLIN-1740: HDFS storage leak in
HBaseResourceStore.deleteResourceImpl() in case of large cell of dictionary
or table snapshot
Repository: kylin
Updated Branches:
refs/heads/master 762992d0a -> 76d60b66c
KYLIN-1740: HDFS storage leak in HBaseResourceStore.deleteResourceImpl() in case of large cell of dictionary or table snapshot
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76d60b66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76d60b66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76d60b66
Branch: refs/heads/master
Commit: 76d60b66c59dfea78734e33fac9c5aa3535bff34
Parents: 762992d
Author: Ma Gang <mg...@163.com>
Authored: Mon Jun 20 12:20:36 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Mon Jun 20 12:22:21 2016 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/HBaseResourceStore.java | 39 ++++++++++++++++----
1 file changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/76d60b66/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 2262482..833c31c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -277,9 +277,28 @@ public class HBaseResourceStore extends ResourceStore {
protected void deleteResourceImpl(String resPath) throws IOException {
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
+ boolean hdfsResourceExist = false;
+ Result result = internalGetFromHTable(table,resPath,true,false);
+ if (result != null) {
+ byte[] value = result.getValue(B_FAMILY, B_COLUMN);
+ if (value != null && value.length == 0) {
+ hdfsResourceExist = true;
+ }
+ }
+
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
table.flushCommits();
+
+ if (hdfsResourceExist) { // remove hdfs cell value
+ Path redirectPath = bigCellHDFSPath(resPath);
+ Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+
+ if (fileSystem.exists(redirectPath)) {
+ fileSystem.delete(redirectPath, true);
+ }
+ }
} finally {
IOUtils.closeQuietly(table);
}
@@ -291,6 +310,15 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ try {
+ return internalGetFromHTable(table,path,fetchContent,fetchTimestamp);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -304,14 +332,9 @@ public class HBaseResourceStore extends ResourceStore {
get.addColumn(B_FAMILY, B_COLUMN_TS);
}
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
- try {
- Result result = table.get(get);
- boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
- return exists ? result : null;
- } finally {
- IOUtils.closeQuietly(table);
- }
+ Result result = table.get(get);
+ boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists()));
+ return exists ? result : null;
}
private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {