You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/26 10:34:59 UTC

[kylin] 01/02: KYLIN-3666 HDFS metadata url not be recognized

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit b4280a13469865f847b6c985d19a98dcd9c59b44
Author: chao long <wa...@qq.com>
AuthorDate: Wed Nov 21 11:29:59 2018 +0800

    KYLIN-3666 HDFS metadata url not be recognized
---
 .../apache/kylin/engine/mr/common/AbstractHadoopJob.java  | 15 +++++++++------
 .../kylin/engine/mr/steps/UpdateDictionaryStep.java       |  3 ++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 8873f30..fe20643 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -490,7 +490,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
 
     public static KylinConfig loadKylinConfigFromHdfs(SerializableConfiguration conf, String uri) {
         HadoopUtil.setCurrentConfiguration(conf.get());
-        return loadKylinConfigFromHdfs(uri);
+        KylinConfig config = loadKylinConfigFromHdfs(uri);
+
+        // This is a bad example where the thread local KylinConfig cannot be auto-closed due to
+        // limitation of MR API. It works because MR task runs its own process. Do not copy.
+        @SuppressWarnings("unused")
+        SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
+        return config;
     }
 
     public static KylinConfig loadKylinConfigFromHdfs(String uri) {
@@ -517,11 +524,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        
-        // This is a bad example where the thread local KylinConfig cannot be auto-closed due to 
-        // limitation of MR API. It works because MR task runs its own process. Do not copy.
-        @SuppressWarnings("unused")
-        SetAndUnsetThreadLocalConfig shouldAutoClose = KylinConfig.setAndUnsetThreadLocalConfig(config);
+
         kylinConfigCache.put(uri, config);
         return config;
     }
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
index bcdb29f..28c567a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateDictionaryStep.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.CubeUpdate;
 import org.apache.kylin.dict.DictionaryInfo;
 import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.job.exception.ExecuteException;
 import org.apache.kylin.job.execution.AbstractExecutable;
@@ -67,7 +68,7 @@ public class UpdateDictionaryStep extends AbstractExecutable {
         final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);
 
         final KylinConfig kylinConfHbase = cube.getConfig();
-        final KylinConfig kylinConfHdfs = KylinConfig.createInstanceFromUri(metadataUrl);
+        final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);
 
         Collections.sort(mergingSegments);