You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/09/06 09:59:46 UTC
[22/50] [abbrv] incubator-kylin git commit: minor,
fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows
minor, fix AbstractHadoopJob.attachKylinPropsAndMetadata() for windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/290d7286
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/290d7286
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/290d7286
Branch: refs/heads/0.7
Commit: 290d728664707ba709ffa2094ab1d506a337c4ab
Parents: 289a834
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Aug 24 15:33:29 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Sun Sep 6 14:37:57 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/AbstractHadoopJob.java | 62 ++++++++++----------
1 file changed, 32 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/290d7286/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index f426618..6ad89d6 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -234,35 +234,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
dumpResources(kylinConfig, metaDir, dumpList);
-
- // hadoop distributed cache
- String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
- logger.info("HDFS meta dir is: " + hdfsMetaDir);
- conf.set("tmpfiles", hdfsMetaDir);
-
- }
-
- protected void cleanupTempConfFile(Configuration conf) {
- String tempMetaFileString = conf.get("tmpfiles");
- logger.info("tempMetaFileString is : " + tempMetaFileString);
- if (tempMetaFileString != null) {
- if (tempMetaFileString.startsWith("file://")) {
- tempMetaFileString = tempMetaFileString.substring("file://".length());
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- try {
- FileUtils.forceDelete(tempMetaFile.getParentFile());
-
- } catch (IOException e) {
- logger.warn("error when deleting " + tempMetaFile, e);
- }
- } else {
- logger.info("" + tempMetaFileString + " does not exist");
- }
- } else {
- logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
- }
- }
+ addToHadoopDistCache(conf, metaDir);
}
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
@@ -293,9 +265,16 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
dumpResources(kylinConfig, metaDir, dumpList);
+ addToHadoopDistCache(conf, metaDir);
+ }
+ private void addToHadoopDistCache(Configuration conf, File metaDir) {
// hadoop distributed cache
- String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ if (hdfsMetaDir.startsWith("/")) // note Path on windows is like "d:/../..."
+ hdfsMetaDir = "file://" + hdfsMetaDir;
+ else
+ hdfsMetaDir = "file:///" + hdfsMetaDir;
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
}
@@ -356,6 +335,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return kylinConfig;
}
+ protected void cleanupTempConfFile(Configuration conf) {
+ String tempMetaFileString = conf.get("tmpfiles");
+ logger.info("tempMetaFileString is : " + tempMetaFileString);
+ if (tempMetaFileString != null) {
+ if (tempMetaFileString.startsWith("file://")) {
+ tempMetaFileString = tempMetaFileString.substring("file://".length());
+ File tempMetaFile = new File(tempMetaFileString);
+ if (tempMetaFile.exists()) {
+ try {
+ FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+ } catch (IOException e) {
+ logger.warn("error when deleting " + tempMetaFile, e);
+ }
+ } else {
+ logger.info("" + tempMetaFileString + " does not exist");
+ }
+ } else {
+ logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
+ }
+ }
+ }
+
public void kill() throws JobException {
if (job != null) {
try {