You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/09/20 22:23:25 UTC
hive git commit: HIVE-20507: Beeline: Add a utility command to
retrieve all uris from beeline-site.xml (Vaibhav Gumashta,
reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 208e202f5 -> 3635c4dba
HIVE-20507: Beeline: Add a utility command to retrieve all uris from beeline-site.xml (Vaibhav Gumashta, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3635c4db
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3635c4db
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3635c4db
Branch: refs/heads/master
Commit: 3635c4dba1613f0aef697b652dbe9e45bb64cb33
Parents: 208e202f
Author: Vaibhav Gumashta <vg...@hortonworks.com>
Authored: Thu Sep 20 15:23:12 2018 -0700
Committer: Vaibhav Gumashta <vg...@hortonworks.com>
Committed: Thu Sep 20 15:23:12 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 57 ++++++++++++++-
.../org/apache/hive/beeline/BeeLineOpts.java | 10 +++
beeline/src/main/resources/BeeLine.properties | 1 +
.../org/apache/hive/jdbc/HiveConnection.java | 73 ++++++++++++++++++++
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 2 +-
5 files changed, 141 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index 29ec2de..e54e818 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -65,6 +65,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.ResourceBundle;
import java.util.ServiceLoader;
@@ -94,6 +95,7 @@ import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils;
import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser;
import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser;
import org.apache.hive.common.util.ShutdownHookManager;
+import org.apache.hive.jdbc.HiveConnection;
import org.apache.hive.jdbc.JdbcUriParseException;
import org.apache.hive.jdbc.Utils;
import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
@@ -389,6 +391,12 @@ public class BeeLine implements Closeable {
.withLongOpt("help")
.withDescription("Display this message")
.create('h'));
+
+ // -getUrlsFromBeelineSite
+ options.addOption(OptionBuilder
+ .withLongOpt("getUrlsFromBeelineSite")
+ .withDescription("Print all urls from beeline-site.xml, if it is present in the classpath")
+ .create());
// Substitution option --hivevar
options.addOption(OptionBuilder
@@ -712,7 +720,7 @@ public class BeeLine implements Closeable {
private boolean isBeeLineOpt(String arg) {
return arg.startsWith("--") && !(HIVE_VAR_PREFIX.equals(arg) || (HIVE_CONF_PREFIX.equals(arg))
- || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg));
+ || "--help".equals(arg) || PROP_FILE_PREFIX.equals(arg) || "--getUrlsFromBeelineSite".equals(arg));
}
}
@@ -846,6 +854,12 @@ public class BeeLine implements Closeable {
getOpts().setHelpAsked(true);
return true;
}
+
+ if (cl.hasOption("getUrlsFromBeelineSite")) {
+ printBeelineSiteUrls();
+ getOpts().setBeelineSiteUrlsAsked(true);
+ return true;
+ }
Properties hiveVars = cl.getOptionProperties("hivevar");
for (String key : hiveVars.stringPropertyNames()) {
@@ -922,6 +936,44 @@ public class BeeLine implements Closeable {
return false;
}
+ private void printBeelineSiteUrls() {
+ BeelineSiteParser beelineSiteParser = getUserBeelineSiteParser();
+ if (!beelineSiteParser.configExists()) {
+ output("No beeline-site.xml in the path", true);
+ }
+ if (beelineSiteParser.configExists()) {
+ // Get the named url from user specific config file if present
+ try {
+ Properties userNamedConnectionURLs = beelineSiteParser.getConnectionProperties();
+ userNamedConnectionURLs.remove(BeelineSiteParser.DEFAULT_NAMED_JDBC_URL_PROPERTY_KEY);
+ StringBuilder sb = new StringBuilder("urls: ");
+ for (Entry<Object, Object> entry : userNamedConnectionURLs.entrySet()) {
+ String urlFromBeelineSite = (String) entry.getValue();
+ if (isZkBasedUrl(urlFromBeelineSite)) {
+ List<String> jdbcUrls = HiveConnection.getAllUrlStrings(urlFromBeelineSite);
+ for (String jdbcUrl : jdbcUrls) {
+ sb.append(jdbcUrl + ", ");
+ }
+ } else {
+ sb.append(urlFromBeelineSite + ", ");
+ }
+ }
+ output(sb.toString(), true);
+ } catch (Exception e) {
+ output(e.getMessage(), true);
+ return;
+ }
+ }
+ }
+
+ private boolean isZkBasedUrl(String urlFromBeelineSite) {
+ String zkJdbcUriParam = ("serviceDiscoveryMode=zooKeeper").toLowerCase();
+ if (urlFromBeelineSite.toLowerCase().contains(zkJdbcUriParam)) {
+ return true;
+ }
+ return false;
+ }
+
private void setHiveConfVar(String key, String val) {
getOpts().getHiveConfVariables().put(key, val);
if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) {
@@ -1068,6 +1120,9 @@ public class BeeLine implements Closeable {
if (getOpts().isHelpAsked()) {
return 0;
}
+ if (getOpts().isBeelineSiteUrlsAsked()) {
+ return 0;
+ }
if (getOpts().getScriptFile() != null) {
return executeFile(getOpts().getScriptFile());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index 062436a..21675dc 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -114,6 +114,7 @@ public class BeeLineOpts implements Completer {
private Map<String, String> hiveVariables = new HashMap<String, String>();
private Map<String, String> hiveConfVariables = new HashMap<String, String>();
private boolean helpAsked;
+ private boolean beelineSiteUrlsAsked;
private String lastConnectedUrl = null;
@@ -679,7 +680,16 @@ public class BeeLineOpts implements Completer {
public boolean isHelpAsked() {
return helpAsked;
}
+
+ public void setBeelineSiteUrlsAsked(boolean beelineSiteUrlsAsked) {
+ this.beelineSiteUrlsAsked = beelineSiteUrlsAsked;
+ }
+
+ public boolean isBeelineSiteUrlsAsked() {
+ return beelineSiteUrlsAsked;
+ }
+
public String getLastConnectedUrl(){
return lastConnectedUrl;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/beeline/src/main/resources/BeeLine.properties
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties
index 30b4ef9..46a5173 100644
--- a/beeline/src/main/resources/BeeLine.properties
+++ b/beeline/src/main/resources/BeeLine.properties
@@ -211,6 +211,7 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \
\ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \
\ marks, slashes, and -- are not allowed; defaults to ;\n \
\ --convertBinaryArrayToString=[true/false] display binary column data as string or as byte array \n \
+\ --getUrlsFromBeelineSite Print all urls from beeline-site.xml, if it is present in the classpath\n \
\ --help display this message\n \
\n \
\ Example:\n \
http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 9f457a2..335995c 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -157,6 +157,79 @@ public class HiveConnection implements java.sql.Connection {
}
return ZooKeeperHiveClientHelper.getDirectParamsList(params);
}
+
+ public static List<String> getAllUrlStrings(String zookeeperBasedHS2Url) throws Exception {
+ List<String> jdbcUrls = new ArrayList<>();
+ List<JdbcConnectionParams> allConnectionParams = getAllUrls(zookeeperBasedHS2Url);
+ for (JdbcConnectionParams cp : allConnectionParams) {
+ String jdbcUrl = makeDirectJDBCUrlFromConnectionParams(cp);
+ if ((jdbcUrl != null) && (!jdbcUrl.isEmpty())) {
+ jdbcUrls.add(jdbcUrl);
+ }
+ }
+ return jdbcUrls;
+ }
+
+ private static String makeDirectJDBCUrlFromConnectionParams(JdbcConnectionParams cp) {
+ // Direct JDBC Url format:
+ // jdbc:hive2://<host1>:<port1>/dbName;sess_var_list?hive_conf_list#hive_var_list
+ StringBuilder url = new StringBuilder("");
+ if (cp != null) {
+ if (cp.getHost() != null) {
+ url.append(cp.getHost());
+ url.append(":");
+ url.append(cp.getPort());
+ url.append("/");
+ url.append(cp.getDbName());
+ // Add session vars
+ if ((cp.getSessionVars() != null) && (!cp.getSessionVars().isEmpty())) {
+ for (Entry<String, String> sessVar : cp.getSessionVars().entrySet()) {
+ if ((sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.SERVICE_DISCOVERY_MODE))
+ || (sessVar.getKey().equalsIgnoreCase(JdbcConnectionParams.ZOOKEEPER_NAMESPACE))) {
+ continue;
+ }
+ url.append(";");
+ url.append(sessVar.getKey());
+ url.append("=");
+ url.append(sessVar.getValue());
+ }
+ }
+ // Add hive confs
+ if ((cp.getHiveConfs() != null) && (!cp.getHiveConfs().isEmpty())) {
+ url.append("?");
+ boolean firstKV = true;
+ for (Entry<String, String> hiveConf : cp.getHiveConfs().entrySet()) {
+ if (!firstKV) {
+ url.append(";");
+ } else {
+ firstKV = false;
+ }
+ url.append(hiveConf.getKey());
+ url.append("=");
+ url.append(hiveConf.getValue());
+ }
+ }
+ // Add hive vars
+ if ((cp.getHiveVars() != null) && (!cp.getHiveVars().isEmpty())) {
+ url.append("#");
+ boolean firstKV = true;
+ for (Entry<String, String> hiveVar : cp.getHiveVars().entrySet()) {
+ if (!firstKV) {
+ url.append(";");
+ } else {
+ firstKV = false;
+ }
+ url.append(hiveVar.getKey());
+ url.append("=");
+ url.append(hiveVar.getValue());
+ }
+ }
+ } else {
+ return url.toString();
+ }
+ }
+ return url.toString();
+ }
public HiveConnection(String uri, Properties info) throws SQLException {
setupLoginTimeout();
http://git-wip-us.apache.org/repos/asf/hive/blob/3635c4db/jdbc/src/java/org/apache/hive/jdbc/Utils.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index 42b3975..852942e 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -114,7 +114,7 @@ public class Utils {
// Use ZooKeeper for indirection while using dynamic service discovery
public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER = "zooKeeper";
public static final String SERVICE_DISCOVERY_MODE_ZOOKEEPER_HA = "zooKeeperHA";
- static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace";
+ public static final String ZOOKEEPER_NAMESPACE = "zooKeeperNamespace";
// Default namespace value on ZooKeeper.
// This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri.
static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";