You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/20 11:38:15 UTC
incubator-kylin git commit: KYLIN-883 Using configurable option for
Hive intermediate tables created by Kylin job
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging d8f491763 -> eee39c8b8
KYLIN-883 Using configurable option for Hive intermediate tables created by Kylin job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/eee39c8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/eee39c8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/eee39c8b
Branch: refs/heads/0.7-staging
Commit: eee39c8b84166a3792624c376919bf6dfff1c285
Parents: d8f4917
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jul 20 17:37:27 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jul 20 17:37:27 2015 +0800
----------------------------------------------------------------------
common/src/main/java/org/apache/kylin/common/KylinConfig.java | 6 ++++++
conf/kylin.properties | 2 ++
examples/test_case_data/sandbox/kylin.properties | 2 ++
job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java | 2 ++
4 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eee39c8b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index bab09ac..cb656c0 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -63,6 +63,8 @@ public class KylinConfig {
public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir";
+ public static final String HIVE_DATABASE_FOR_INTERMEDIATE_TABLE = "kylin.job.hive.database.for.intermediatetable";
+
public static final String HIVE_TABLE_LOCATION_PREFIX = "hive.table.location.";
public static final String KYLIN_JOB_REMOTE_CLI_PASSWORD = "kylin.job.remote.cli.password";
@@ -621,6 +623,10 @@ public class KylinConfig {
this.storageUrl = storageUrl;
}
+ public String getHiveDatabaseForIntermediateTable() {
+ return this.getOptional(HIVE_DATABASE_FOR_INTERMEDIATE_TABLE, "default");
+ }
+
public void setRunAsRemoteCommand(String v) {
kylinConfig.setProperty(KYLIN_JOB_RUN_AS_REMOTE_CMD, v);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eee39c8b/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index aa232bb..cee23b2 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -53,6 +53,8 @@ kylin.job.concurrent.max.limit=10
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
+# Hive database name for putting the intermediate flat tables
+kylin.job.hive.database.for.intermediatetable=default
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eee39c8b/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index caacb6e..767dcb2 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -53,6 +53,8 @@ kylin.job.concurrent.max.limit=10
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
+# Hive database name for putting the intermediate flat tables
+kylin.job.hive.database.for.intermediatetable=default
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/eee39c8b/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java b/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
index e1b7cf9..2ebc012 100644
--- a/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/AbstractJobBuilder.java
@@ -60,6 +60,7 @@ public abstract class AbstractJobBuilder {
protected AbstractExecutable createIntermediateHiveTableStep(IJoinedFlatTableDesc intermediateTableDesc, String jobId) {
+ final String useDatabaseHql = "USE " + engineConfig.getConfig().getHiveDatabaseForIntermediateTable() + ";";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobId);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, getJobWorkingDir(jobId), jobId);
String insertDataHqls;
@@ -73,6 +74,7 @@ public abstract class AbstractJobBuilder {
ShellExecutable step = new ShellExecutable();
StringBuffer buf = new StringBuffer();
buf.append("hive -e \"");
+ buf.append(useDatabaseHql + "\n");
buf.append(dropTableHql + "\n");
buf.append(createTableHql + "\n");
buf.append(insertDataHqls + "\n");