You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/05 05:23:22 UTC
[34/67] [abbrv] kylin git commit: KYLIN-2648 make
kylin.env.hdfs-working-dir qualified and absolute
KYLIN-2648 make kylin.env.hdfs-working-dir qualified and absolute
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3a79c82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3a79c82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3a79c82
Branch: refs/heads/master
Commit: e3a79c8276123bb9fa3a9de9bfc653a81607d392
Parents: 6bfc6d2
Author: Li Yang <li...@apache.org>
Authored: Sat May 27 17:03:09 2017 +0800
Committer: hongbin ma <ma...@kyligence.io>
Committed: Sat May 27 17:08:06 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 40 +++++++++++++-------
.../apache/kylin/common/KylinConfigTest.java | 7 ++++
.../java/org/apache/kylin/rest/DebugTomcat.java | 10 ++---
3 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ad08108..854ffbd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -29,6 +29,9 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.lock.DistributedLockFactory;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -184,21 +187,33 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.env", "DEV");
}
+ private String cachedHdfsWorkingDirectory;
+
public String getHdfsWorkingDirectory() {
+ if (cachedHdfsWorkingDirectory != null)
+ return cachedHdfsWorkingDirectory;
+
String root = getRequired("kylin.env.hdfs-working-dir");
- if (!root.endsWith("/")) {
- root += "/";
- }
-
- // make sure path qualified
- if (!root.contains("://")) {
- if (!root.startsWith("/"))
- root = "hdfs:///" + root;
- else
- root = "hdfs://" + root;
+ Path path = new Path(root);
+ if (path.isAbsolute() == false)
+ throw new IllegalArgumentException("kylin.env.hdfs-working-dir must be absolute, but got " + root);
+
+ // make sure path is qualified
+ try {
+ FileSystem fs = path.getFileSystem(new Configuration());
+ path = fs.makeQualified(path);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
-
- return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
+
+ // append metadata-url prefix
+ root = new Path(path, StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).toString();
+
+ if (root.endsWith("/") == false)
+ root += "/";
+
+ cachedHdfsWorkingDirectory = root;
+ return cachedHdfsWorkingDirectory;
}
// ============================================================================
@@ -1028,5 +1043,4 @@ abstract public class KylinConfigBase implements Serializable {
public boolean isWebCrossDomainEnabled() {
return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 6027af3..fcbdd36 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -124,4 +124,11 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
}
}).start();
}
+
+ @Test
+ public void testHdfsWorkingDir() {
+ KylinConfig conf = KylinConfig.getInstanceFromEnv();
+ String hdfsWorkingDirectory = conf.getHdfsWorkingDirectory();
+ assertTrue(hdfsWorkingDirectory.startsWith("file:/"));
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3a79c82/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index cd6768e..d87f1f9 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -18,6 +18,10 @@
package org.apache.kylin.rest;
+import java.io.File;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
@@ -27,10 +31,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
-import java.io.File;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
public class DebugTomcat {
public static void setupDebugEnv() {
@@ -101,7 +101,7 @@ public class DebugTomcat {
public static void main(String[] args) throws Exception {
setupDebugEnv();
-
+
int port = 7070;
if (args.length >= 1) {
port = Integer.parseInt(args[0]);