You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/26 14:56:34 UTC
[26/34] kylin git commit: minor,
create job work dir with Materialize Hive View in Lookup Tables job.
minor, create job work dir with Materialize Hive View in Lookup Tables job.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ae39b5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ae39b5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ae39b5b
Branch: refs/heads/master
Commit: 0ae39b5bb1cb05d3660ffb94c125cd8e85b91902
Parents: 7d5fb85
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/HiveMRInput.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ae39b5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive