You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/26 11:20:49 UTC
[23/33] kylin git commit: minor,
create job work dir with Materialize Hive View in Lookup Tables job.
minor, create job work dir with Materialize Hive View in Lookup Tables job.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de9b5286
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de9b5286
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de9b5286
Branch: refs/heads/sync
Commit: de9b528697729df97bffa26112c2e761e4cd5634
Parents: 327bd76
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:01:08 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/HiveMRInput.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/de9b5286/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive