You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by kw...@apache.org on 2018/03/05 20:02:35 UTC
bigtop git commit: BIGTOP-3007: hive pkg and charm opts for zookeeper
Repository: bigtop
Updated Branches:
refs/heads/master c9cb18fbc -> 97d2e578b
BIGTOP-3007: hive pkg and charm opts for zookeeper
Closes #344
Signed-off-by: Kevin W Monroe <ke...@canonical.com>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/97d2e578
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/97d2e578
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/97d2e578
Branch: refs/heads/master
Commit: 97d2e578b6a83ead40a0467f4793583a14bad89e
Parents: c9cb18f
Author: jamesbeedy <ja...@gmail.com>
Authored: Tue Feb 27 09:19:06 2018 -0800
Committer: Kevin W Monroe <ke...@canonical.com>
Committed: Mon Mar 5 12:16:58 2018 -0600
----------------------------------------------------------------------
.../modules/hadoop_hive/manifests/init.pp | 2 +
.../modules/hadoop_hive/templates/hive-site.xml | 14 +++++
.../ignite_hadoop/templates/hive-site.xml | 2 +-
.../src/charm/hive/layer-hive/layer.yaml | 1 +
.../layer-hive/lib/charms/layer/bigtop_hive.py | 63 +++++++++++++++-----
.../src/charm/hive/layer-hive/metadata.yaml | 2 +
.../src/charm/hive/layer-hive/reactive/hive.py | 12 +++-
7 files changed, 79 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp
index d03427c..35c06179 100644
--- a/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp
@@ -47,6 +47,8 @@ class hadoop_hive {
class common_config ($hbase_master = "",
$hbase_zookeeper_quorum = "",
+ $hive_zookeeper_quorum = "",
+ $hive_support_concurrency = false,
$kerberos_realm = "",
$metastore_uris = "",
$server2_thrift_port = "10000",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml
index 6e4b4b6..ff1fb92 100644
--- a/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml
@@ -42,6 +42,20 @@
</property>
<% end %>
+<% if @hive_zookeeper_quorum != "" %>
+<property>
+ <name>hive.zookeeper.quorum</name>
+ <value><%= @hive_zookeeper_quorum %></value>
+ <description>Zookeeper quorum used by Hive's Table Lock Manager</description>
+</property>
+<% end %>
+
+<property>
+ <name>hive.support.concurrency</name>
+ <value><%= @hive_support_concurrency %></value>
+ <description>Enable Hive's Table Lock Manager Service</description>
+</property>
+
<property>
<name>hive.execution.engine</name>
<value><%= @hive_execution_engine %></value>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml
index 1f58898..753c501 100644
--- a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml
+++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml
@@ -30,4 +30,4 @@
<name>hive.rpc.query.plan</name>
<value>true</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-packages/src/charm/hive/layer-hive/layer.yaml
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/hive/layer-hive/layer.yaml b/bigtop-packages/src/charm/hive/layer-hive/layer.yaml
index 5be7ac9..0a5ba23 100644
--- a/bigtop-packages/src/charm/hive/layer-hive/layer.yaml
+++ b/bigtop-packages/src/charm/hive/layer-hive/layer.yaml
@@ -5,6 +5,7 @@ includes:
- 'interface:hbase'
- 'interface:hive'
- 'interface:mysql'
+ - 'interface:zookeeper'
options:
basic:
packages:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py b/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py
index c7d9cc6..1aa0fdc 100755
--- a/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py
+++ b/bigtop-packages/src/charm/hive/layer-hive/lib/charms/layer/bigtop_hive.py
@@ -25,7 +25,7 @@ class Hive(object):
self.dist_config = utils.DistConfig(
data=layer.options('apache-bigtop-base'))
- def install(self, hbase=None):
+ def install(self, hbase=None, zk_units=None):
'''
Trigger the Bigtop puppet recipe that handles the Hive service.
'''
@@ -36,23 +36,37 @@ class Hive(object):
# Prep config
roles = ['hive-client', 'hive-metastore', 'hive-server2']
metastore = "thrift://{}:9083".format(hookenv.unit_private_ip())
+
if hbase:
roles.append('hive-hbase')
hb_connect = "{}:{}".format(hbase['host'], hbase['master_port'])
- zk_connect = hbase['zk_connect']
+ zk_hbase_connect = hbase['zk_connect']
else:
hb_connect = ""
- zk_connect = ""
+ zk_hbase_connect = ""
+
+ if zk_units:
+ hive_support_concurrency = True
+ zk_hive_connect = self.get_zk_connect(zk_units)
+ else:
+ hive_support_concurrency = False
+ zk_hive_connect = ""
override = {
'hadoop_hive::common_config::hbase_master': hb_connect,
- 'hadoop_hive::common_config::hbase_zookeeper_quorum': zk_connect,
+ 'hadoop_hive::common_config::hbase_zookeeper_quorum':
+ zk_hbase_connect,
+ 'hadoop_hive::common_config::hive_zookeeper_quorum':
+ zk_hive_connect,
+ 'hadoop_hive::common_config::hive_support_concurrency':
+ hive_support_concurrency,
'hadoop_hive::common_config::metastore_uris': metastore,
'hadoop_hive::common_config::server2_thrift_port':
self.dist_config.port('hive-thrift'),
'hadoop_hive::common_config::server2_thrift_http_port':
self.dist_config.port('hive-thrift-web'),
}
+
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles, overrides=override)
bigtop.trigger_puppet()
@@ -60,7 +74,16 @@ class Hive(object):
# Bigtop doesn't create a hive-env.sh, but we need it for heap config
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
if not hive_env.exists():
- (self.dist_config.path('hive_conf') / 'hive-env.sh.template').copy(hive_env)
+ (self.dist_config.path('hive_conf') / 'hive-env.sh.template').copy(
+ hive_env)
+
+ def get_zk_connect(self, zk_units):
+ zks = []
+ for unit in zk_units:
+ ip = utils.resolve_private_address(unit['host'])
+ zks.append(ip)
+ zks.sort()
+ return ",".join(zks)
def configure_hive(self):
'''
@@ -69,35 +92,44 @@ class Hive(object):
config = hookenv.config()
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
utils.re_edit_in_place(hive_env, {
- r'.*export HADOOP_HEAPSIZE *=.*': 'export HADOOP_HEAPSIZE=%s' % config['heap'],
+ r'.*export HADOOP_HEAPSIZE *=.*':
+ 'export HADOOP_HEAPSIZE=%s' % config['heap'],
})
def configure_remote_db(self, mysql):
hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml'
- jdbc_url = "jdbc:mysql://{}:{}/{}?createDatabaseIfNotExist=true".format(
- mysql.host(), mysql.port(), mysql.database()
- )
+ jdbc_url = \
+ "jdbc:mysql://{}:{}/{}?createDatabaseIfNotExist=true".format(
+ mysql.host(), mysql.port(), mysql.database()
+ )
with utils.xmlpropmap_edit_in_place(hive_site) as props:
props['javax.jdo.option.ConnectionURL'] = jdbc_url
props['javax.jdo.option.ConnectionUserName'] = mysql.user()
props['javax.jdo.option.ConnectionPassword'] = mysql.password()
- props['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver"
+ props['javax.jdo.option.ConnectionDriverName'] = \
+ "com.mysql.jdbc.Driver"
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
utils.re_edit_in_place(hive_env, {
- r'.*export HIVE_AUX_JARS_PATH *=.*': 'export HIVE_AUX_JARS_PATH=/usr/share/java/mysql-connector-java.jar',
+ r'.*export HIVE_AUX_JARS_PATH *=.*':
+ ('export HIVE_AUX_JARS_PATH='
+ '/usr/share/java/mysql-connector-java.jar'),
})
# Now that we have db connection info, init our schema (only once)
remote_db = hookenv.remote_service_name()
if not unitdata.kv().get('hive.schema.initialized.%s' % remote_db):
- tool_path = "{}/bin/schematool".format(self.dist_config.path('hive'))
- utils.run_as('ubuntu', tool_path, '-initSchema', '-dbType', 'mysql')
+ tool_path = "{}/bin/schematool".format(
+ self.dist_config.path('hive'))
+ utils.run_as(
+ 'ubuntu', tool_path, '-initSchema', '-dbType', 'mysql')
unitdata.kv().set('hive.schema.initialized.%s' % remote_db, True)
unitdata.kv().flush(True)
def configure_local_db(self):
- local_url = 'jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true'
+ local_url = \
+ ('jdbc:derby:;databaseName='
+ '/var/lib/hive/metastore/metastore_db;create=true')
local_driver = 'org.apache.derby.jdbc.EmbeddedDriver'
hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml'
with utils.xmlpropmap_edit_in_place(hive_site) as props:
@@ -108,7 +140,8 @@ class Hive(object):
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
utils.re_edit_in_place(hive_env, {
- r'.*export HIVE_AUX_JARS_PATH *=.*': '# export HIVE_AUX_JARS_PATH=',
+ r'.*export HIVE_AUX_JARS_PATH *=.*':
+ '# export HIVE_AUX_JARS_PATH=',
})
def restart(self):
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml b/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml
index 998dd72..0c814c1 100644
--- a/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml
+++ b/bigtop-packages/src/charm/hive/layer-hive/metadata.yaml
@@ -17,3 +17,5 @@ requires:
interface: mysql
hbase:
interface: hbase
+ zookeeper:
+ interface: zookeeper
http://git-wip-us.apache.org/repos/asf/bigtop/blob/97d2e578/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py b/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py
index 0deebd4..bf4ce2b 100644
--- a/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py
+++ b/bigtop-packages/src/charm/hive/layer-hive/reactive/hive.py
@@ -36,6 +36,7 @@ def report_status():
database_joined = is_state('database.connected')
database_ready = is_state('database.available')
hive_installed = is_state('hive.installed')
+
if not hadoop_joined:
hookenv.status_set('blocked',
'waiting for relation to hadoop plugin')
@@ -78,9 +79,17 @@ def install_hive(hadoop):
else:
hbserver = None
+ # Get zookeeper connection dict if it's available
+ if is_state('zookeeper.ready'):
+ zk = RelationBase.from_state('zookeeper.ready')
+ zks = zk.zookeepers()
+ else:
+ zks = None
+
# Use this to determine if we need to reinstall
deployment_matrix = {
'hbase': hbserver,
+ 'zookeepers': zks
}
# Handle nuances when installing versus re-installing
@@ -99,8 +108,9 @@ def install_hive(hadoop):
hookenv.status_set('maintenance', '{} hive'.format(prefix))
hookenv.log("{} hive with: {}".format(prefix, deployment_matrix))
+
hive = Hive()
- hive.install(hbase=hbserver)
+ hive.install(hbase=hbserver, zk_units=zks)
hive.restart()
hive.open_ports()
set_state('hive.installed')