You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:22 UTC
[19/50] [abbrv] kylin git commit: minor,
add switch table-dir-create-first for build cube with hive step.
minor, add switch table-dir-create-first for build cube with hive step.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce1775a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce1775a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce1775a5
Branch: refs/heads/sync
Commit: ce1775a5ab0e775bce6685e7801f114a4aff9b9b
Parents: 543124d
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../main/java/org/apache/kylin/source/hive/HiveMRInput.java | 8 +++++---
.../java/org/apache/kylin/source/hive/HiveMRInputTest.java | 2 +-
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5045b2f..6d1b7f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.source.hive.sparksql-beeline-params", "");
}
+ public boolean getHiveTableDirCreateFirst() {
+ return Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", "false"));
+ }
+
public String getFlatHiveTableClusterByDictColumn() {
return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
+ if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) {
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ }
return jobWorkingDir;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
try {
KylinConfig kylinConfig = mock(KylinConfig.class);
KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
defaultChainedExecutable.setId(UUID.randomUUID().toString());