You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/26 10:34:59 UTC
[kylin] 01/02: KYLIN-3666 HDFS metadata url not be recognized
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b4280a13469865f847b6c985d19a98dcd9c59b44
Author: chao long <wa...@qq.com>
AuthorDate: Wed Nov 21 11:29:59 2018 +0800
KYLIN-3666 HDFS metadata url not be recognized
---
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 15 +++++++++------
.../kylin/engine/mr/steps/UpdateDictionaryStep.java | 3 ++-
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 8873f30..fe20643 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -490,7 +490,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
public static KylinConfig loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) {
HadoopUtil.setCurrentConfiguration(conf.get());
- return loadKylinConfigFromHdfs(uri);
+ KylinConfig config = loadKylinConfigFromHdfs(uri);
+
+ // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
+ // limitation of MR API. It works because MR task runs its own process. Do not copy.
+ @SuppressWarnings("unused")
+ SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
+ return config;
}
public static KylinConfig loadKylinConfigFromHdfs(String uri) {
@@ -517,11 +524,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
- // limitation of MR API. It works because MR task runs its own process. Do not copy.
- @SuppressWarnings("unused")
- SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
kylinConfigCache.put(uri, config);
return config;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
index bcdb29f..28c567a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -67,7 +68,7 @@ public class UpdateDictionaryStep extends AbstractExecutable {
final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);
final KylinConfig kylinConfHbase = cube.getConfig();
- final KylinConfig kylinConfHdfs = KylinConfig.createInstanceFromUri(metadataUrl);
+ final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
Collections.sort(mergingSegments);