You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/11/18 03:39:09 UTC
[15/30] incubator-kylin git commit: KYLIN-1099 Fix test case for
windows
KYLIN-1099 Fix test case for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6b6be739
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6b6be739
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6b6be739
Branch: refs/heads/1.x-HBase1.1.3
Commit: 6b6be7397872a51606a3dca8b6b67bbd43058493
Parents: a441237
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Nov 11 10:49:25 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Nov 11 10:49:25 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/HadoopUtil.java | 19 +++++++++++++++----
.../kylin/dict/lookup/FileTableReader.java | 2 ++
2 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b6be739/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java b/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 3255b32..483a7a1 100644
--- a/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -65,12 +65,23 @@ public class HadoopUtil {
public static URI makeURI(String filePath) {
try {
- return new URI(filePath);
+ return new URI(fixWindowsPath(filePath));
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Cannot create FileSystem from URI: " + filePath, e);
}
}
+ public static String fixWindowsPath(String path) {
+ // fix windows path
+ if (path.startsWith("file://") && !path.startsWith("file:///") && path.contains(":\\")) {
+ path = path.replace("file://", "file:///");
+ }
+ if (path.startsWith("file:///")) {
+ path = path.replace('\\', '/');
+ }
+ return path;
+ }
+
public static String makeQualifiedPathInHadoopCluster(String path) {
try {
FileSystem fs = FileSystem.get(getCurrentConfiguration());
@@ -93,10 +104,10 @@ public class HadoopUtil {
Configuration conf = new Configuration();
return healSickConfig(conf);
}
-
+
public static Configuration newHBaseConfiguration(String url) {
Configuration conf = HBaseConfiguration.create(getCurrentConfiguration());
-
+
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
@@ -106,7 +117,7 @@ public class HadoopUtil {
if (StringUtils.isNotEmpty(hbaseClusterFs)) {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hbaseClusterFs);
}
-
+
// reduce rpc retry
conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6b6be739/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
index 5ef7731..a6de5df 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/lookup/FileTableReader.java
@@ -172,6 +172,7 @@ public class FileTableReader implements TableReader {
Text value;
SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException {
+ path = HadoopUtil.fixWindowsPath(path);
reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path)));
key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
value = new Text();
@@ -196,6 +197,7 @@ public class FileTableReader implements TableReader {
BufferedReader reader;
CsvRowReader(FileSystem fs, String path) throws IOException {
+ path = HadoopUtil.fixWindowsPath(path);
FSDataInputStream in = fs.open(new Path(path));
reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
}