You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/01/11 19:50:35 UTC
[1/2] hive git commit: HIVE-12813 : LLAP: issues in setup,
shutdown (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 abe594ec7 -> b277f8ae7
refs/heads/master fad307719 -> c278860a9
HIVE-12813 : LLAP: issues in setup, shutdown (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c278860a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c278860a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c278860a
Branch: refs/heads/master
Commit: c278860a941b1014eea6d3a85cfa0c5eb16d49e5
Parents: fad3077
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jan 11 10:45:44 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jan 11 10:45:44 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 7 ++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 60 ++++++++++++--------
.../hive/llap/daemon/impl/LlapDaemon.java | 6 +-
.../impl/LlapDaemonProtocolServerImpl.java | 1 +
.../daemon/services/impl/LlapWebServices.java | 2 -
5 files changed, 47 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 479fa46..eedd205 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2409,9 +2409,10 @@ public class HiveConf extends Configuration {
"ZooKeeper for ZooKeeper SecretManager."),
LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "",
"ZooKeeper connection string for ZooKeeper SecretManager."),
- LLAP_SECURITY_ACL("hive.llap.daemon.service.acl", "*", "The ACL for LLAP daemon."),
- LLAP_MANAGEMENT_ACL("hive.llap.management.service.acl", "*",
- "The ACL for LLAP daemon management."),
+ // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
+ // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
+ LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."),
+ LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP daemon management."),
// Hadoop DelegationTokenManager default is 1 week.
LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", "14d",
new TimeValidator(TimeUnit.SECONDS),
http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 4b330f4..0d54558 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -18,10 +18,9 @@
package org.apache.hadoop.hive.llap.cli;
+import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -49,6 +48,10 @@ public class LlapServiceDriver {
protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
private static final String[] DEFAULT_AUX_CLASSES = new String[] {
"org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.hbase.HBaseSerDe" };
+ private static final String[] NEEDED_CONFIGS = {
+ "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
+ private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" };
+
private final Configuration conf;
@@ -115,17 +118,16 @@ public class LlapServiceDriver {
FileSystem fs = FileSystem.get(conf);
FileSystem lfs = FileSystem.getLocal(conf).getRawFileSystem();
- String[] neededConfig =
- { "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
-
// needed so that the file is actually loaded into configuration.
- for (String f : neededConfig) {
+ for (String f : NEEDED_CONFIGS) {
conf.addResource(f);
if (conf.getResource(f) == null) {
throw new Exception("Unable to find required config file: " + f);
}
}
-
+ for (String f : OPTIONAL_CONFIGS) {
+ conf.addResource(f);
+ }
conf.reloadConfiguration();
if (options.getName() != null) {
@@ -251,22 +253,14 @@ public class LlapServiceDriver {
Path confPath = new Path(tmpDir, "conf");
lfs.mkdirs(confPath);
- for (String f : neededConfig) {
- if (f.equals("llap-daemon-site.xml")) {
- FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
-
- Configuration copy = resolve(conf, "llap-daemon-site.xml");
-
- for (Entry<Object, Object> props : options.getConfig().entrySet()) {
- // overrides
- copy.set((String) props.getKey(), (String) props.getValue());
- }
-
- copy.writeXml(confStream);
- confStream.close();
- } else {
- // they will be file:// URLs
- lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
+ for (String f : NEEDED_CONFIGS) {
+ copyConfig(options, lfs, confPath, f);
+ }
+ for (String f : OPTIONAL_CONFIGS) {
+ try {
+ copyConfig(options, lfs, confPath, f);
+ } catch (Throwable t) {
+ LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage());
}
}
@@ -312,4 +306,24 @@ public class LlapServiceDriver {
LOG.debug("Exiting successfully");
}
}
+
+ private void copyConfig(
+ LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException {
+ if (f.equals("llap-daemon-site.xml")) {
+ FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
+
+ Configuration copy = resolve(conf, "llap-daemon-site.xml");
+
+ for (Entry<Object, Object> props : options.getConfig().entrySet()) {
+ // overrides
+ copy.set((String) props.getKey(), (String) props.getValue());
+ }
+
+ copy.writeXml(confStream);
+ confStream.close();
+ } else {
+ // they will be file:// URLs
+ lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index b3057c3..917ff32 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -258,7 +258,11 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
public void shutdown() {
LOG.info("LlapDaemon shutdown invoked");
if (llapDaemonInfoBean != null) {
- MBeans.unregister(llapDaemonInfoBean);
+ try {
+ MBeans.unregister(llapDaemonInfoBean);
+ } catch (Throwable ex) {
+ LOG.info("Error unregistering the bean; ignoring", ex);
+ }
}
if (pauseMonitor != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
index f87fffe..45ca906 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
@@ -215,6 +215,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService
return server;
}
+
@Override
public GetTokenResponseProto getDelegationToken(RpcController controller,
GetTokenRequestProto request) throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hive/blob/c278860a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
index ed51f3c..e233b41 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
@@ -43,7 +43,6 @@ public class LlapWebServices extends AbstractService {
@Override
public void serviceInit(Configuration conf) {
-
this.conf = new Configuration(conf);
this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
@@ -55,7 +54,6 @@ public class LlapWebServices extends AbstractService {
@Override
public void serviceStart() throws Exception {
String bindAddress = "0.0.0.0";
- Configuration conf = getConfig();
if (UserGroupInformation.isSecurityEnabled()
&& HiveConf.getBoolVar(conf, ConfVars.LLAP_WEB_AUTO_AUTH)) {
conf.set("hadoop.http.authentication.type", "kerberos");
[2/2] hive git commit: HIVE-12813 : LLAP: issues in setup,
shutdown (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Posted by se...@apache.org.
HIVE-12813 : LLAP: issues in setup, shutdown (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b277f8ae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b277f8ae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b277f8ae
Branch: refs/heads/branch-2.0
Commit: b277f8ae77d058979e95c9258c78dafa266607fc
Parents: abe594e
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jan 11 10:45:44 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jan 11 10:50:18 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 7 ++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 60 ++++++++++++--------
.../hive/llap/daemon/impl/LlapDaemon.java | 6 +-
.../impl/LlapDaemonProtocolServerImpl.java | 1 +
.../daemon/services/impl/LlapWebServices.java | 2 -
5 files changed, 47 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b277f8ae/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index caf7842..33d4bc0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2397,9 +2397,10 @@ public class HiveConf extends Configuration {
"ZooKeeper for ZooKeeper SecretManager."),
LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "",
"ZooKeeper connection string for ZooKeeper SecretManager."),
- LLAP_SECURITY_ACL("hive.llap.daemon.service.acl", "*", "The ACL for LLAP daemon."),
- LLAP_MANAGEMENT_ACL("hive.llap.management.service.acl", "*",
- "The ACL for LLAP daemon management."),
+ // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
+ // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
+ LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."),
+ LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP daemon management."),
// Hadoop DelegationTokenManager default is 1 week.
LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", "14d",
new TimeValidator(TimeUnit.SECONDS),
http://git-wip-us.apache.org/repos/asf/hive/blob/b277f8ae/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index 4b330f4..0d54558 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -18,10 +18,9 @@
package org.apache.hadoop.hive.llap.cli;
+import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -49,6 +48,10 @@ public class LlapServiceDriver {
protected static final Logger LOG = LoggerFactory.getLogger(LlapServiceDriver.class.getName());
private static final String[] DEFAULT_AUX_CLASSES = new String[] {
"org.apache.hive.hcatalog.data.JsonSerDe", "org.apache.hadoop.hive.hbase.HBaseSerDe" };
+ private static final String[] NEEDED_CONFIGS = {
+ "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
+ private static final String[] OPTIONAL_CONFIGS = { "ssl-server.xml" };
+
private final Configuration conf;
@@ -115,17 +118,16 @@ public class LlapServiceDriver {
FileSystem fs = FileSystem.get(conf);
FileSystem lfs = FileSystem.getLocal(conf).getRawFileSystem();
- String[] neededConfig =
- { "tez-site.xml", "hive-site.xml", "llap-daemon-site.xml", "core-site.xml" };
-
// needed so that the file is actually loaded into configuration.
- for (String f : neededConfig) {
+ for (String f : NEEDED_CONFIGS) {
conf.addResource(f);
if (conf.getResource(f) == null) {
throw new Exception("Unable to find required config file: " + f);
}
}
-
+ for (String f : OPTIONAL_CONFIGS) {
+ conf.addResource(f);
+ }
conf.reloadConfiguration();
if (options.getName() != null) {
@@ -251,22 +253,14 @@ public class LlapServiceDriver {
Path confPath = new Path(tmpDir, "conf");
lfs.mkdirs(confPath);
- for (String f : neededConfig) {
- if (f.equals("llap-daemon-site.xml")) {
- FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
-
- Configuration copy = resolve(conf, "llap-daemon-site.xml");
-
- for (Entry<Object, Object> props : options.getConfig().entrySet()) {
- // overrides
- copy.set((String) props.getKey(), (String) props.getValue());
- }
-
- copy.writeXml(confStream);
- confStream.close();
- } else {
- // they will be file:// URLs
- lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
+ for (String f : NEEDED_CONFIGS) {
+ copyConfig(options, lfs, confPath, f);
+ }
+ for (String f : OPTIONAL_CONFIGS) {
+ try {
+ copyConfig(options, lfs, confPath, f);
+ } catch (Throwable t) {
+ LOG.info("Error getting an optional config " + f + "; ignoring: " + t.getMessage());
}
}
@@ -312,4 +306,24 @@ public class LlapServiceDriver {
LOG.debug("Exiting successfully");
}
}
+
+ private void copyConfig(
+ LlapOptions options, FileSystem lfs, Path confPath, String f) throws IOException {
+ if (f.equals("llap-daemon-site.xml")) {
+ FSDataOutputStream confStream = lfs.create(new Path(confPath, f));
+
+ Configuration copy = resolve(conf, "llap-daemon-site.xml");
+
+ for (Entry<Object, Object> props : options.getConfig().entrySet()) {
+ // overrides
+ copy.set((String) props.getKey(), (String) props.getValue());
+ }
+
+ copy.writeXml(confStream);
+ confStream.close();
+ } else {
+ // they will be file:// URLs
+ lfs.copyFromLocalFile(new Path(conf.getResource(f).toString()), confPath);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b277f8ae/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index b3057c3..917ff32 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -258,7 +258,11 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
public void shutdown() {
LOG.info("LlapDaemon shutdown invoked");
if (llapDaemonInfoBean != null) {
- MBeans.unregister(llapDaemonInfoBean);
+ try {
+ MBeans.unregister(llapDaemonInfoBean);
+ } catch (Throwable ex) {
+ LOG.info("Error unregistering the bean; ignoring", ex);
+ }
}
if (pauseMonitor != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b277f8ae/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
index f87fffe..45ca906 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
@@ -215,6 +215,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService
return server;
}
+
@Override
public GetTokenResponseProto getDelegationToken(RpcController controller,
GetTokenRequestProto request) throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hive/blob/b277f8ae/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
index ed51f3c..e233b41 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
@@ -43,7 +43,6 @@ public class LlapWebServices extends AbstractService {
@Override
public void serviceInit(Configuration conf) {
-
this.conf = new Configuration(conf);
this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
@@ -55,7 +54,6 @@ public class LlapWebServices extends AbstractService {
@Override
public void serviceStart() throws Exception {
String bindAddress = "0.0.0.0";
- Configuration conf = getConfig();
if (UserGroupInformation.isSecurityEnabled()
&& HiveConf.getBoolVar(conf, ConfVars.LLAP_WEB_AUTO_AUTH)) {
conf.set("hadoop.http.authentication.type", "kerberos");