You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/04/03 03:15:28 UTC
[kylin] 03/08: KYLIN-3277 Kylin should override hiveconf settings
when connecting to hive using jdbc
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch sync
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 25b4b3bc06b30a4030f4949d97a05566bcbddca4
Author: ycq <yc...@meitu.com>
AuthorDate: Wed Mar 7 17:10:38 2018 +0800
KYLIN-3277 Kylin should override hiveconf settings when connecting to hive using jdbc
Signed-off-by: nichunen <ch...@kyligence.io>
---
.../apache/kylin/common/util/HiveCmdBuilder.java | 47 +---------
.../kylin/common/util/HiveConfigurationUtil.java | 101 +++++++++++++++++++++
.../kylin/source/hive/BeelineHiveClient.java | 16 +++-
3 files changed, 115 insertions(+), 49 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index fcfc598..9a9c65b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -46,12 +46,12 @@ public class HiveCmdBuilder {
}
private KylinConfig kylinConfig;
- final private Map<String, String> hiveConfProps = new HashMap<>();
+ final private Map<String, String> hiveConfProps;
final private ArrayList<String> statements = Lists.newArrayList();
public HiveCmdBuilder() {
kylinConfig = KylinConfig.getInstanceFromEnv();
- loadHiveConfiguration();
+ hiveConfProps = HiveConfigurationUtil.loadHiveConfiguration();
}
public String build() {
@@ -153,47 +153,4 @@ public class HiveCmdBuilder {
return build();
}
- private void loadHiveConfiguration() {
-
- File hiveConfFile;
- String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
- String path = System.getProperty(KylinConfig.KYLIN_CONF);
-
- if (StringUtils.isNotEmpty(path)) {
- hiveConfFile = new File(path, hiveConfFileName);
- } else {
- path = KylinConfig.getKylinHome();
- if (StringUtils.isEmpty(path)) {
- logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME);
- return;
- }
- hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
- }
-
- if (!hiveConfFile.exists()) {
- throw new RuntimeException("Missing config file: " + hiveConfFile.getAbsolutePath());
- }
-
- String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
-
- try {
- File file = new File(fileUrl);
- if (file.exists()) {
- DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder = factory.newDocumentBuilder();
- Document doc = builder.parse(file);
- NodeList nl = doc.getElementsByTagName("property");
- hiveConfProps.clear();
- for (int i = 0; i < nl.getLength(); i++) {
- String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
- String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
- if (!key.equals("tmpjars")) {
- hiveConfProps.put(key, value);
- }
- }
- }
- } catch (Exception e) {
- throw new RuntimeException("Failed to parse hive conf file ", e);
- }
- }
}
\ No newline at end of file
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java
new file mode 100644
index 0000000..a8cba61
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.kylin.common.util.HiveCmdBuilder.HIVE_CONF_FILENAME;
+
+/**
+ * @author ycq
+ * @since 2018-03-05
+ */
+public class HiveConfigurationUtil {
+
+ private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HiveConfigurationUtil.class);
+ private static final String HIVE_CONF_PREFIX = "hiveconf:";
+
+ public static Properties loadHiveJDBCProperties() {
+ Map<String, String> hiveConfiguration = loadHiveConfiguration();
+ Properties ret = new Properties();
+ for (Map.Entry<String, String> entry : hiveConfiguration.entrySet()) {
+ ret.put(HIVE_CONF_PREFIX + entry.getKey(), entry.getValue());
+ }
+ return ret;
+ }
+
+ public static Map<String, String> loadHiveConfiguration() {
+ Map<String, String> hiveConfProps = new HashMap<>();
+ File hiveConfFile;
+ String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml");
+ String path = System.getProperty(KylinConfig.KYLIN_CONF);
+
+ if (StringUtils.isNotEmpty(path)) {
+ hiveConfFile = new File(path, hiveConfFileName);
+ } else {
+ path = KylinConfig.getKylinHome();
+ if (StringUtils.isEmpty(path)) {
+ logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME);
+ return hiveConfProps;
+ }
+ hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName);
+ }
+
+ if (!hiveConfFile.exists()) {
+ throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml");
+ }
+
+ String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath());
+
+ try {
+ File file = new File(fileUrl);
+ if (file.exists()) {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
+ NodeList nl = doc.getElementsByTagName("property");
+ hiveConfProps.clear();
+ for (int i = 0; i < nl.getLength(); i++) {
+ String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
+ String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
+ if (!key.equals("tmpjars")) {
+ hiveConfProps.put(key, value);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to parse hive conf file ", e);
+ }
+ return hiveConfProps;
+ }
+
+}
+
+
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 747b1bb..593ad96 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -26,25 +26,30 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
+import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.DBUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.kylin.common.util.HiveConfigurationUtil;
public class BeelineHiveClient implements IHiveClient {
+ private static final String HIVE_AUTH_USER = "user";
+ private static final String HIVE_AUTH_PASSWD = "password";
private Connection cnct;
private Statement stmt;
private DatabaseMetaData metaData;
+
public BeelineHiveClient(String beelineParams) {
if (StringUtils.isEmpty(beelineParams)) {
throw new IllegalArgumentException("BeelineParames cannot be empty");
}
String[] splits = StringUtils.split(beelineParams);
- String url = null, username = null, password = null;
+ String url = "", username = "", password = "";
for (int i = 0; i < splits.length; i++) {
if ("-u".equals(splits[i])) {
url = stripQuotes(splits[i + 1]);
@@ -56,13 +61,16 @@ public class BeelineHiveClient implements IHiveClient {
password = stripQuotes(splits[i + 1]);
}
}
- this.init(url, username, password);
+ Properties jdbcProperties = HiveConfigurationUtil.loadHiveJDBCProperties();
+ jdbcProperties.put(HIVE_AUTH_PASSWD, password);
+ jdbcProperties.put(HIVE_AUTH_USER, username);
+ this.init(url, jdbcProperties);
}
- private void init(String url, String username, String password) {
+ private void init(String url, Properties hiveProperties) {
try {
Class.forName("org.apache.hive.jdbc.HiveDriver");
- cnct = DriverManager.getConnection(url, username, password);
+ cnct = DriverManager.getConnection(url, hiveProperties);
stmt = cnct.createStatement();
metaData = cnct.getMetaData();
} catch (SQLException | ClassNotFoundException e) {
--
To stop receiving notification emails like this one, please contact
liyang@apache.org.