You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/05/24 10:42:40 UTC
[09/50] [abbrv] kylin git commit: KYLIN-2577,
refine hive configurations during cubing steps
KYLIN-2577, refine hive configurations during cubing steps
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c152c51
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c152c51
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c152c51
Branch: refs/heads/KYLIN-2606
Commit: 7c152c515ce6cebe380c4e1b7c16d109192308db
Parents: 0061900
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu May 18 11:05:34 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu May 18 15:27:36 2017 +0800
----------------------------------------------------------------------
build/bin/load-hive-conf.sh | 2 +-
.../kylin/common/util/HiveCmdBuilder.java | 56 +++++++++++++++++++-
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
.../kylin/job/engine/JobEngineConfig.java | 43 ---------------
.../source/hive/CreateFlatHiveTableStep.java | 3 --
.../apache/kylin/source/hive/HiveMRInput.java | 4 --
6 files changed, 57 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/build/bin/load-hive-conf.sh
----------------------------------------------------------------------
diff --git a/build/bin/load-hive-conf.sh b/build/bin/load-hive-conf.sh
index a5046f0..a42cd0c 100644
--- a/build/bin/load-hive-conf.sh
+++ b/build/bin/load-hive-conf.sh
@@ -11,5 +11,5 @@ hive_conf_prop="${KYLIN_HOME}/logs/hive_props"
rm -rf ${hive_conf_prop}
export ENABLE_CHECK_ENV=false
${dir}/kylin.sh org.apache.kylin.tool.HiveConfigCLI ${hive_conf_dir} ${hive_conf_prop}
-[[ 0 == $? ]] || quit "Error, can not parse ${hive_conf_dir} and can not apply it to hive relevant check."
+[[ 0 == $? ]] || quit "Can not parse xml file: ${hive_conf_dir}, please check it."
hive_conf_properties=`cat ${hive_conf_prop}`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index 255867a..2f6b9a0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -27,17 +27,25 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
+ public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+
public enum HiveClientMode {
CLI, BEELINE
}
@@ -50,6 +58,7 @@ public class HiveCmdBuilder {
public HiveCmdBuilder() {
kylinConfig = KylinConfig.getInstanceFromEnv();
clientMode = HiveClientMode.valueOf(kylinConfig.getHiveClientMode().toUpperCase());
+ loadHiveConfiguration();
}
public String build() {
@@ -75,8 +84,8 @@ public class HiveCmdBuilder {
bw.newLine();
}
buf.append("beeline ");
- buf.append(parseProps());
buf.append(kylinConfig.getHiveBeelineParams());
+ buf.append(parseProps());
buf.append(" -f ");
buf.append(tmpHql.getAbsolutePath());
buf.append(";ret_code=$?;rm -f ");
@@ -123,6 +132,7 @@ public class HiveCmdBuilder {
}
public void setHiveConfProps(Map<String, String> hiveConfProps) {
+ this.hiveConfProps.clear();
this.hiveConfProps.putAll(hiveConfProps);
}
@@ -144,4 +154,48 @@ public class HiveCmdBuilder {
public String toString() {
return build();
}
+
+ private void loadHiveConfiguration() {
+
+ File hiveConfFile;
+ String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+ String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+ if (StringUtils.isNotEmpty(path)) {
+ hiveConfFile = new File(path, hiveConfFileName);
+ } else {
+ path = KylinConfig.getKylinHome();
+ if (StringUtils.isEmpty(path)) {
+ logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME);
+ return;
+ }
+ hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
+ }
+
+ if (hiveConfFile == null || !hiveConfFile.exists()) {
+ throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml");
+ }
+
+ String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
+
+ try {
+ File file = new File(fileUrl);
+ if (file.exists()) {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
+ NodeList nl = doc.getElementsByTagName("property");
+ hiveConfProps.clear();
+ for (int i = 0; i < nl.getLength(); i++) {
+ String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+ String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+ if (!key.equals("tmpjars")) {
+ hiveConfProps.put(key, value);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to parse hive conf file ", e);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index d847575..d69d4d2 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -75,7 +75,7 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("beeline -u jdbc_url -f"));
+ assertTrue(cmd.startsWith("beeline -u jdbc_url"));
String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim();
hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 260f817..c9ac583 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -20,19 +20,12 @@ package org.apache.kylin.job.engine;
import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
/**
* @author ysong1
@@ -40,7 +33,6 @@ import org.w3c.dom.NodeList;
public class JobEngineConfig {
private static final Logger logger = LoggerFactory.getLogger(JobEngineConfig.class);
public static final String HADOOP_JOB_CONF_FILENAME = "kylin_job_conf";
- public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
public static final String DEFAUL_JOB_CONF_SUFFIX = "";
public static final String IN_MEM_JOB_CONF_SUFFIX = "inmem";
@@ -102,41 +94,6 @@ public class JobEngineConfig {
return path;
}
- public String getHiveConfFilePath() {
- String hiveConfFile = (HIVE_CONF_FILENAME + ".xml");
-
- File jobConfig = getJobConfig(hiveConfFile);
- if (jobConfig == null || !jobConfig.exists()) {
-
- logger.error("fail to locate " + HIVE_CONF_FILENAME + ".xml");
- throw new RuntimeException("fail to locate " + HIVE_CONF_FILENAME + ".xml");
- }
- return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath());
- }
-
- public Map<String, String> getHivePropsFromFile() {
- Map<String, String> props = new HashMap<>();
- try {
- File file = new File(getHiveConfFilePath());
- if (file.exists()) {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder = factory.newDocumentBuilder();
- Document doc = builder.parse(file);
- NodeList nl = doc.getElementsByTagName("property");
- for (int i = 0; i < nl.getLength(); i++) {
- String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
- String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
- if (!key.equals("tmpjars")) {
- props.put(key, value);
- }
- }
- }
- } catch (Exception e) {
- throw new RuntimeException("Failed to parse hive conf file ", e);
- }
- return props;
- }
-
// there should be no setters
private final KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index 9e13df8..f28f3c7 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -26,7 +26,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.common.PatternedLogger;
-import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -43,8 +42,6 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
protected void createFlatHiveTable(KylinConfig config) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- final JobEngineConfig jobEngineConfig = new JobEngineConfig(config);
- hiveCmdBuilder.setHiveConfProps(jobEngineConfig.getHivePropsFromFile());
hiveCmdBuilder.overwriteHiveProps(config.getHiveConfigOverride());
hiveCmdBuilder.addStatement(getInitStatement());
hiveCmdBuilder.addStatement(getCreateTableStatement());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c152c51/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 6c542ab..e388074 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -257,7 +257,6 @@ public class HiveMRInput implements IMRInput {
private void redistributeTable(KylinConfig config, int numReducers) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- hiveCmdBuilder.setHiveConfProps(new JobEngineConfig(config).getHivePropsFromFile());
hiveCmdBuilder.addStatement(getInitStatement());
hiveCmdBuilder.addStatement("set mapreduce.job.reduces=" + numReducers + ";\n");
hiveCmdBuilder.addStatement("set hive.merge.mapredfiles=false;\n");
@@ -375,13 +374,10 @@ public class HiveMRInput implements IMRInput {
final String hiveTable = this.getIntermediateTableIdentity();
if (config.isHiveKeepFlatTable() == false && StringUtils.isNotEmpty(hiveTable)) {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- hiveCmdBuilder.setHiveConfProps(new JobEngineConfig(config).getHivePropsFromFile());
hiveCmdBuilder.addStatement("USE " + config.getHiveDatabaseForIntermediateTable() + ";");
hiveCmdBuilder.addStatement("DROP TABLE IF EXISTS " + hiveTable + ";");
-
config.getCliCommandExecutor().execute(hiveCmdBuilder.build());
output.append("Hive table " + hiveTable + " is dropped. \n");
-
rmdirOnHDFS(getExternalDataPath());
output.append("Hive table " + hiveTable + " external data path " + getExternalDataPath() + " is deleted. \n");
}