You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/23 12:20:45 UTC
[07/10] kylin git commit: Add HDFSResourceStore to ResourceStore
Add HDFSResourceStore to ResourceStore
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c33e38f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c33e38f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c33e38f
Branch: refs/heads/master-hbase0.98
Commit: 4c33e38f3e6c33438ef3bb830875b8ed74ffac5b
Parents: d3276e2
Author: xiefan46 <95...@qq.com>
Authored: Mon Jan 23 10:51:12 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 23 16:23:56 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 5 +++++
.../kylin/storage/hdfs/ITHDFSResourceStoreTest.java | 13 ++++---------
.../apache/kylin/storage/hdfs/ITLockManagerTest.java | 14 +++++++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 3 ++-
.../apache/kylin/storage/hdfs/HDFSResourceStore.java | 4 +++-
5 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 9549569..25a0801 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -75,6 +75,11 @@ abstract public class ResourceStore {
} catch (Throwable e) {
logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString());
}
+ try {
+ knownImpl.add(ClassUtil.forName("org.apache.kylin.storage.hdfs.HDFSResourceStore", ResourceStore.class));
+ } catch (Throwable e) {
+ logger.warn("Failed to load HDFSResourceStore impl class: " + e.toString());
+ }
}
return knownImpl;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
index 27d8a3c..47afb01 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java
@@ -19,17 +19,12 @@
package org.apache.kylin.storage.hdfs;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceStoreTest;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
-/**
- * Created by xiefan on 17-1-10.
- */
public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
KylinConfig kylinConfig;
@@ -45,13 +40,13 @@ public class ITHDFSResourceStoreTest extends HBaseMetadataTestCase {
this.cleanupTestMetadata();
}
-
@Test
public void testResourceStoreBasic() throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- ResourceStore store = new HDFSResourceStore(config);
+ String oldUrl = kylinConfig.getMetadataUrl();
+ kylinConfig.setProperty("kylin.metadata.url", "kylin_default_instance@hdfs");
+ HDFSResourceStore store = new HDFSResourceStore(kylinConfig);
ResourceStoreTest.testAStore(store);
+ kylinConfig.setProperty("kylin.metadata.url", oldUrl);
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
index 2b58d30..a6cf43b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java
@@ -27,6 +27,8 @@ import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
@@ -58,6 +60,8 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
private static final int REPETITIONS = QTY * 10;
+ private static final Logger logger = LoggerFactory.getLogger(ITLockManagerTest.class);
+
@Before
public void setup() throws Exception {
this.createTestMetadata();
@@ -66,7 +70,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
zkClient = CuratorFrameworkFactory.newClient(zkConnection, retryPolicy);
zkClient.start();
manager = new LockManager(kylinConfig, lockRootPath);
- System.out.println("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
+ logger.info("nodes in lock root : " + zkClient.getChildren().forPath(lockRootPath));
}
@@ -75,7 +79,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
this.cleanupTestMetadata();
zkClient.delete().deletingChildrenIfNeeded().forPath(lockRootPath);
List<String> nodes = zkClient.getChildren().forPath("/");
- System.out.println("nodes in zk after delete : " + nodes);
+ logger.info("nodes in zk after delete : " + nodes);
manager.close();
}
@@ -85,7 +89,7 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
ResourceLock lock = manager.getLock("/dictionary/numberdict.json");
lock.acquire();
manager.releaseLock(lock);
- System.out.println(zkClient.getChildren().forPath(lockRootPath + "/dictionary"));
+ logger.info(zkClient.getChildren().forPath(lockRootPath + "/dictionary").toString());
List<String> nodes = zkClient.getChildren().forPath(lockRootPath + "/dictionary");
assertEquals(1, nodes.size());
assertEquals("numberdict.json", nodes.get(0));
@@ -193,10 +197,10 @@ public class ITLockManagerTest extends HBaseMetadataTestCase {
throw new IllegalStateException(clientName + " could not acquire the lock");
}
try {
- System.out.println(clientName + " has the lock");
+ logger.info(clientName + " has the lock");
resource.use();
} finally {
- System.out.println(clientName + " releasing the lock");
+ logger.info(clientName + " releasing the lock");
lock.release(); // always release the lock in a finally block
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 170e351..84670dd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -82,7 +82,8 @@ public class HBaseResourceStore extends ResourceStore {
int cut = metadataUrl.indexOf('@');
tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
+ if (!hbaseUrl.equals("hbase"))
+ throw new IOException("Can not create HBaseResourceStore. Url not match. Url:" + hbaseUrl);
createHTableIfNeeded(getAllInOneTableName());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c33e38f/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
index a746a97..bdd3917 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java
@@ -57,12 +57,14 @@ public class HDFSResourceStore extends ResourceStore {
private LockManager lockManager;
- //public for test. Normal should be protected
public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
super(kylinConfig);
String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut);
+ String hdfsUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+ if(!hdfsUrl.equals("hdfs"))
+ throw new IOException("Can not create HDFSResourceStore. Url not match. Url:" + hdfsUrl);
metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME;
logger.info("meta dir name :" + metaDirName);
createMetaFolder(metaDirName, kylinConfig);