You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/05/23 11:36:56 UTC
[03/16] kylin git commit: KYLIN-2459 fix implementation
KYLIN-2459 fix implementation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/429dfbdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/429dfbdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/429dfbdc
Branch: refs/heads/master
Commit: 429dfbdc93371e8df2095e35298425ccab8ac7de
Parents: 1b864f0
Author: Yang Li <li...@apache.org>
Authored: Sat May 20 19:52:30 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Sat May 20 21:35:36 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 36 ++++++----------
.../kylin/common/persistence/ResourceStore.java | 45 ++++++--------------
.../common/persistence/ResourceStoreTest.java | 11 +----
.../common/util/LocalFileMetadataTestCase.java | 7 +--
4 files changed, 32 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index df3baf4..8930129 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -218,6 +218,16 @@ abstract public class KylinConfigBase implements Serializable {
return getMetadataUrl().getIdentifier();
}
+ public Map<String, String> getResourceStoreImpls() {
+ Map<String, String> r = Maps.newLinkedHashMap();
+ // ref constants in ISourceAware
+ r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
+ r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
+ r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
+ r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html
+ return r;
+ }
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers", //
new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
@@ -456,6 +466,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.job.advanced-flat-table.class");
}
+ public String getJobTrackingURLPattern() {
+ return getOptional("kylin.job.tracking-url-pattern", "");
+ }
+
// ============================================================================
// SOURCE.HIVE
// ============================================================================
@@ -970,26 +984,4 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
- //ResourceStore Impl
- /*public String getResourceStoreImpl() {
- return getOptional("kylin.metadata.default-resource-store-impl", "org.apache.kylin.storage.hbase.HBaseResourceStore");
- }*/
-
- public Map<String, String> getResourceStoreImpls() {
- Map<String, String> r = Maps.newLinkedHashMap();
- // ref constants in ISourceAware
- r.put("", "org.apache.kylin.common.persistence.FileResourceStore");
- r.put("hbase", "org.apache.kylin.storage.hbase.HBaseResourceStore");
- r.put("hdfs", "org.apache.kylin.storage.hdfs.HDFSResourceStore");
- r.putAll(getPropertiesByPrefix("kylin.resource.store.provider."));
- return r;
- }
-
- public String getResourceStoreImpl() {
- return getResourceStoreImpls().get(getMetadataUrl().getScheme());
- }
-
- public String getJobTrackingURLPattern() {
- return getOptional("kylin.job.tracking-url-pattern", "");
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 0565c66..7fb93e7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
@@ -68,46 +69,24 @@ abstract public class ResourceStore {
public static final String CUBE_STATISTICS_ROOT = "/cube_statistics";
public static final String BAD_QUERY_RESOURCE_ROOT = "/bad_query";
- protected static final String DEFAULT_STORE_NAME = "kylin_metadata";
-
public static final String METASTORE_UUID_TAG = "/UUID";
private static final ConcurrentMap<KylinConfig, ResourceStore> CACHE = new ConcurrentHashMap<KylinConfig, ResourceStore>();
- private static final ArrayList<Class<? extends ResourceStore>> knownImpl = new ArrayList<Class<? extends ResourceStore>>();
-
- private static ArrayList<Class<? extends ResourceStore>> getKnownImpl(KylinConfig kylinConfig) {
- if (knownImpl.isEmpty()) {
- knownImpl.add(FileResourceStore.class);
- try {
- String implName = kylinConfig.getResourceStoreImpl();
- knownImpl.add(ClassUtil.forName(implName, ResourceStore.class));
- } catch (Throwable e) {
- logger.warn("Failed to load HBaseResourceStore impl class: " + e.toString());
- }
- }
- return knownImpl;
- }
-
private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
- List<Throwable> es = new ArrayList<Throwable>();
- logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
- for (Class<? extends ResourceStore> cls : getKnownImpl(kylinConfig)) {
- try {
- ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
- if (!store.exists(METASTORE_UUID_TAG)) {
- store.putResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0, StringEntity.serializer);
- }
- return store;
- } catch (Throwable e) {
-
- es.add(e);
+ StorageURL metadataUrl = kylinConfig.getMetadataUrl();
+ logger.info("Using metadata url " + metadataUrl + " for resource store");
+ String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme());
+ try {
+ Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class);
+ ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
+ if (!store.exists(METASTORE_UUID_TAG)) {
+ store.putResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0, StringEntity.serializer);
}
+ return store;
+ } catch (Throwable e) {
+ throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e);
}
- for (Throwable exceptionOrError : es) {
- logger.error("Create new store instance failed ", exceptionOrError);
- }
- throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
}
public static ResourceStore getStore(KylinConfig kylinConfig) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
index 8aff1a3..91a9dfd 100644
--- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.NavigableSet;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,13 +44,13 @@ public class ResourceStoreTest {
public static void testAStore(String url, KylinConfig kylinConfig) throws Exception {
String oldUrl = replaceMetadataUrl(kylinConfig, url);
- testAStore(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig));
+ testAStore(ResourceStore.getStore(kylinConfig));
replaceMetadataUrl(kylinConfig, oldUrl);
}
public static void testPerformance(String url, KylinConfig kylinConfig) throws Exception {
String oldUrl = replaceMetadataUrl(kylinConfig, url);
- testPerformance(getStoreByName(kylinConfig.getResourceStoreImpl(), kylinConfig));
+ testPerformance(ResourceStore.getStore(kylinConfig));
replaceMetadataUrl(kylinConfig, oldUrl);
}
@@ -205,10 +204,4 @@ public class ResourceStoreTest {
return oldUrl;
}
- private static ResourceStore getStoreByName(String storeName, KylinConfig kylinConfig) throws Exception {
- Class<? extends ResourceStore> cls = ClassUtil.forName(storeName, ResourceStore.class);
- ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
- return store;
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/429dfbdc/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index a44a55d..67f39b4 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -63,11 +63,12 @@ public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
}
public static void cleanAfterClass() {
- String tempTestMetadataUrl = LOCALMETA_TEMP_DATA;
+ File directory = new File(LOCALMETA_TEMP_DATA);
try {
- FileUtils.deleteDirectory(new File(tempTestMetadataUrl));
+ FileUtils.deleteDirectory(directory);
} catch (IOException e) {
- throw new IllegalStateException("Can't delete directory " + tempTestMetadataUrl, e);
+ if (directory.exists() && directory.list().length > 0)
+ throw new IllegalStateException("Can't delete directory " + directory, e);
}
staticCleanupTestMetadata();
}