You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/06 07:51:31 UTC
kylin git commit: KYLIN-1839 bug fix
Repository: kylin
Updated Branches:
refs/heads/master b162e6c4e -> 13eeb685b
KYLIN-1839 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/13eeb685
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/13eeb685
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/13eeb685
Branch: refs/heads/master
Commit: 13eeb685b71d2d2f0ed3f59c2ccde0a4ff3e8f25
Parents: b162e6c
Author: Yang Li <li...@apache.org>
Authored: Sun Nov 6 15:50:49 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Nov 6 15:51:13 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/AbstractHadoopJob.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/13eeb685/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 21bb10e..1fd1631 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -295,9 +295,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
continue;
}
FileSystem fs;
- if (hdfs.exists(p)) {
+ if (exists(hdfs, p)) {
fs = hdfs;
- } else if (localfs.exists(p)) {
+ } else if (exists(localfs, p)) {
fs = localfs;
} else {
logger.warn("The directory of kylin dependency '" + fileName + "' does not exist, skip");
@@ -386,6 +386,15 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return classpath;
}
+ private static boolean exists(FileSystem fs, Path p) throws IOException {
+ try {
+ return fs.exists(p);
+ } catch (IllegalArgumentException ex) {
+ // can happen when FS mismatch
+ return false;
+ }
+ }
+
public static int addInputDirs(String input, Job job) throws IOException {
int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
logger.info("Number of added folders:" + folderNum);
@@ -401,7 +410,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
FileSystem fs = FileSystem.get(job.getConfiguration());
Path path = new Path(inp);
- if (!fs.exists(path)) {
+ if (!exists(fs, path)) {
logger.warn("Path not exist:" + path.toString());
continue;
}