You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/08/15 13:34:35 UTC
[kylin] 11/11: KYLIN-4121 Cleanup hive view intermediate tables
after job be finished
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 75179ab5d96a574e32807491edcf06d0a0292b07
Author: rupengwang <wa...@live.cn>
AuthorDate: Fri Aug 2 21:18:52 2019 +0800
KYLIN-4121 Cleanup hive view intermediate tables after job be finished
---
.../org/apache/kylin/source/hive/GarbageCollectionStep.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
index ed86513..c541751 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/GarbageCollectionStep.java
@@ -47,8 +47,6 @@ public class GarbageCollectionStep extends AbstractExecutable {
StringBuffer output = new StringBuffer();
try {
output.append(cleanUpIntermediateFlatTable(config));
- // don't drop view to avoid concurrent issue
- //output.append(cleanUpHiveViewIntermediateTable(config));
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
return ExecuteResult.createError(e);
@@ -57,6 +55,7 @@ public class GarbageCollectionStep extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
}
+ //clean up both hive intermediate flat table and view table
private String cleanUpIntermediateFlatTable(KylinConfig config) throws IOException {
String quoteCharacter = FlatTableSqlQuoteUtils.getQuote();
StringBuffer output = new StringBuffer();
@@ -94,9 +93,14 @@ public class GarbageCollectionStep extends AbstractExecutable {
setParam("oldHiveTables", StringUtil.join(tableIdentity, ","));
}
+ //get intermediate fact table and lookup table(if exists)
private List<String> getIntermediateTables() {
List<String> intermediateTables = Lists.newArrayList();
- String[] tables = StringUtil.splitAndTrim(getParam("oldHiveTables"), ",");
+ String hiveTables = getParam("oldHiveTables");
+ if (this.getParams().containsKey("oldHiveViewIntermediateTables")) {
+ hiveTables += ("," + getParam("oldHiveViewIntermediateTables"));
+ }
+ String[] tables = StringUtil.splitAndTrim(hiveTables, ",");
for (String t : tables) {
intermediateTables.add(t);
}