You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/06/04 12:58:40 UTC
kylin git commit: KYLIN-1758 should set hive conf when creating
materialized tables
Repository: kylin
Updated Branches:
refs/heads/master 6f6dcf00c -> 4018e6671
KYLIN-1758 should set hive conf when creating materialized tables
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4018e667
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4018e667
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4018e667
Branch: refs/heads/master
Commit: 4018e6671f6f4e23f4493286881fa47f302c6372
Parents: 6f6dcf0
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jun 4 20:58:30 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Jun 4 20:58:30 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/JoinedFlatTable.java | 25 +++++++++-----------
.../kylin/provision/BuildIIWithStream.java | 7 +-----
.../apache/kylin/source/hive/HiveMRInput.java | 20 ++++------------
3 files changed, 16 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index d625ad7..24032fa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -55,15 +55,16 @@ public class JoinedFlatTable {
return storageDfsDir + "/" + intermediateTableDesc.getTableName();
}
- public static String generateHiveSetStatements(JobEngineConfig engineConfig) throws IOException {
+ public static String generateHiveSetStatements(JobEngineConfig engineConfig) {
StringBuilder buffer = new StringBuilder();
- File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath());
- if (hadoopPropertiesFile.exists()) {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- try {
+ try {
+ File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath());
+
+ if (hadoopPropertiesFile.exists()) {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder;
+ Document doc;
builder = factory.newDocumentBuilder();
doc = builder.parse(hadoopPropertiesFile);
NodeList nl = doc.getElementsByTagName("property");
@@ -74,14 +75,10 @@ public class JoinedFlatTable {
buffer.append("SET " + name + "=" + value + ";\n");
}
}
-
- } catch (ParserConfigurationException e) {
- throw new IOException(e);
- } catch (SAXException e) {
- throw new IOException(e);
}
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to parse hive conf file ", e);
}
-
return buffer.toString();
}
@@ -114,7 +111,7 @@ public class JoinedFlatTable {
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) throws IOException {
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc intermediateTableDesc, JobEngineConfig engineConfig) {
StringBuilder sql = new StringBuilder();
sql.append(generateHiveSetStatements(engineConfig));
sql.append("INSERT OVERWRITE TABLE " + intermediateTableDesc.getTableName() + " " + generateSelectDataStatement(intermediateTableDesc) + ";").append("\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
index c0f6f63..1a335c6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildIIWithStream.java
@@ -148,12 +148,7 @@ public class BuildIIWithStream {
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid));
String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig);
ShellExecutable step = new ShellExecutable();
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4018e667/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 0c33c11..76dda34 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -120,7 +120,6 @@ public class HiveMRInput implements IMRInput {
}
jobFlow.addTask(createCountHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId(), rowCountOutputDir));
jobFlow.addTask(createFlatHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId(), cubeName, rowCountOutputDir));
-// jobFlow.addTask(createFlatHiveTableStep(conf, flatHiveTableDesc, jobFlow.getId()));
AbstractExecutable task = createLookupHiveViewMaterializationStep(jobFlow.getId());
if(task != null) {
jobFlow.addTask(task);
@@ -131,11 +130,7 @@ public class HiveMRInput implements IMRInput {
final ShellExecutable step = new ShellExecutable();
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- try {
- hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
- } catch (IOException e) {
- throw new RuntimeException("Failed to generate hive set statements for createCountHiveTableStep", e);
- }
+ hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
hiveCmdBuilder.addStatement("set hive.exec.compress.output=false;\n");
hiveCmdBuilder.addStatement(JoinedFlatTable.generateCountDataStatement(flatTableDesc, rowCountOutputDir));
@@ -168,6 +163,7 @@ public class HiveMRInput implements IMRInput {
}
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
hiveCmdBuilder.addStatement(useDatabaseHql);
+ hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
for(TableDesc lookUpTableDesc : lookupViewsTables) {
if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) {
StringBuilder createIntermediateTableHql = new StringBuilder();
@@ -190,21 +186,13 @@ public class HiveMRInput implements IMRInput {
public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName, String rowCountOutputDir) {
StringBuilder hiveInitBuf = new StringBuilder();
- try {
- hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
- } catch (IOException e) {
- throw new RuntimeException("Failed to generate hive set statements for RedistributeFlatHiveTableStep", e);
- }
+ hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf);
- } catch (IOException e) {
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.", e);
- }
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf);
CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
step.setInitStatement(hiveInitBuf.toString());