You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/04/24 03:49:29 UTC
[kylin] 07/07: KYLIN-3340, enhance hql check
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b1c8944471558afe63d61b6578c494decbae53cc
Author: Cheng Wang <ch...@kyligence.io>
AuthorDate: Mon Apr 16 15:17:29 2018 +0800
KYLIN-3340, enhance hql check
---
.../org/apache/kylin/source/hive/HiveMRInput.java | 24 +++++++++++++---------
.../apache/kylin/source/hive/HiveMRInputTest.java | 19 +++++++++++++++++
2 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 38656cf..a96f4d5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -238,16 +238,8 @@ public class HiveMRInput implements IMRInput {
String identity = lookUpTableDesc.getIdentity();
String intermediate = lookUpTableDesc.getMaterializedName();
if (lookUpTableDesc.isView()) {
- StringBuilder createIntermediateTableHql = new StringBuilder();
- createIntermediateTableHql.append("DROP TABLE IF EXISTS " + intermediate + ";\n");
- createIntermediateTableHql
- .append("CREATE EXTERNAL TABLE IF NOT EXISTS " + intermediate + " LIKE " + identity + "\n");
- createIntermediateTableHql.append("LOCATION '" + jobWorkingDir + "/" + intermediate + "';\n");
- createIntermediateTableHql
- .append("ALTER TABLE " + intermediate + " SET TBLPROPERTIES('auto.purge'='true');\n");
- createIntermediateTableHql
- .append("INSERT OVERWRITE TABLE " + intermediate + " SELECT * FROM " + identity + ";\n");
- hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
+ String materializeViewHql = materializeViewHql(intermediate, identity, jobWorkingDir);
+ hiveCmdBuilder.addStatement(materializeViewHql);
hiveViewIntermediateTables = hiveViewIntermediateTables + intermediate + ";";
}
}
@@ -259,6 +251,18 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ // each append must be a complete hql.
+ public static String materializeViewHql(String viewName, String tableName, String jobWorkingDir) {
+ StringBuilder createIntermediateTableHql = new StringBuilder();
+ createIntermediateTableHql.append("DROP TABLE IF EXISTS " + viewName + ";\n");
+ createIntermediateTableHql.append("CREATE EXTERNAL TABLE IF NOT EXISTS " + viewName + " LIKE " + tableName
+ + " LOCATION '" + jobWorkingDir + "/" + viewName + "';\n");
+ createIntermediateTableHql.append("ALTER TABLE " + viewName + " SET TBLPROPERTIES('auto.purge'='true');\n");
+ createIntermediateTableHql
+ .append("INSERT OVERWRITE TABLE " + viewName + " SELECT * FROM " + tableName + ";\n");
+ return createIntermediateTableHql.toString();
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 8df3174..a81cc21 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -56,4 +56,23 @@ public class HiveMRInputTest {
}
}
+ @Test
+ public void testMaterializeViewHql() {
+ final int viewSize = 2;
+ String[] mockedViewNames = { "mockedView1", "mockedView2" };
+ String[] mockedTalbeNames = { "mockedTable1", "mockedTable2" };
+ String mockedWorkingDir = "mockedWorkingDir";
+
+ StringBuilder hqls = new StringBuilder();
+ for (int i = 0; i < viewSize; i++) {
+ String hql = HiveMRInput.BatchCubingInputSide.materializeViewHql(mockedViewNames[i], mockedTalbeNames[i],
+ mockedWorkingDir);
+ hqls.append(hql);
+ }
+
+ for (String sub : hqls.toString().split("\n")) {
+ Assert.assertTrue(sub.endsWith(";"));
+ }
+ }
+
}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.