You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by kw...@apache.org on 2017/10/20 16:22:33 UTC
bigtop git commit: BIGTOP-2910: zeppelin charm: support bigtop upgrade
Repository: bigtop
Updated Branches:
refs/heads/master 08aa3a267 -> 4ad7f6bd2
BIGTOP-2910: zeppelin charm: support bigtop upgrade
Fixes #298
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4ad7f6bd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4ad7f6bd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4ad7f6bd
Branch: refs/heads/master
Commit: 4ad7f6bd291c467b3a4af76ae7baba5eb13fcacb
Parents: 08aa3a2
Author: Kevin W Monroe <ke...@canonical.com>
Authored: Thu Oct 19 21:21:50 2017 +0000
Committer: Kevin W Monroe <ke...@canonical.com>
Committed: Fri Oct 20 11:21:16 2017 -0500
----------------------------------------------------------------------
.../charm/zeppelin/layer-zeppelin/actions.yaml | 9 ++-
.../zeppelin/layer-zeppelin/actions/reinstall | 59 +++++++++++++++++
.../lib/charms/layer/bigtop_zeppelin.py | 70 +++++++++++---------
.../layer-zeppelin/reactive/zeppelin.py | 50 ++++++++++----
.../layer-zeppelin/tests/02-zeppelin-smoke.py | 2 +-
.../tests/03-zeppelin-spark-smoke.py | 2 +-
.../layer-zeppelin/tests/04-zeppelin-config.py | 48 ++++++++++++++
7 files changed, 191 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml
index 7ce817e..375a8ca 100644
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions.yaml
@@ -1,7 +1,10 @@
-smoke-test:
+reinstall:
description: >
- Verify that Zeppelin is working by running one of
- the example notebook paragraphs, using the REST server.
+ Reinstall Zeppelin with the version available in the repo.
restart:
description: >
Restart Zeppelin.
+smoke-test:
+ description: >
+ Verify that Zeppelin is working by running one of the example notebook
+ paragraphs, using the REST server.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall
new file mode 100755
index 0000000..7ac8e75
--- /dev/null
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/reinstall
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+sys.path.append('lib')
+
+from charmhelpers.core import hookenv, unitdata # noqa: E402
+from charms.layer.apache_bigtop_base import Bigtop, get_package_version # noqa: E402
+from charms.reactive import is_state # noqa: E402
+
+
+def fail(msg):
+ hookenv.action_set({'outcome': 'failure'})
+ hookenv.action_fail(msg)
+ sys.exit()
+
+
+if not is_state('bigtop.version.changed'):
+ fail('No Bigtop version changes were found; nothing to reinstall.')
+
+if not unitdata.kv().get('zeppelin.version.repo', False):
+ fail('Charm is not prepared to run the reinstall action.')
+
+# Call the base reinstall method and ensure zeppelin packages
+# are removed prior to reinstalling new versions via puppet apply.
+bigtop = Bigtop()
+result = bigtop.reinstall_repo_packages(remove_pkgs='zeppelin')
+
+if bigtop.check_bigtop_repo_package('zeppelin'):
+ # Ruh roh. We expect this to be None since we just did a reinstall
+ # with the current repo. There should be no different version available.
+ fail('Unexpected zeppelin version found after reinstalling')
+
+if result == 'success':
+ # Set appropriate status output
+ app_version = get_package_version('zeppelin') or 'unknown'
+ hookenv.application_version_set(app_version)
+ hookenv.status_set('active', 'reinstall was successful')
+
+ # Remove our version unitdata and report success
+ unitdata.kv().unset('zeppelin.version.repo')
+ hookenv.action_set({'outcome': 'success'})
+else:
+ fail('Reinstall failed; hiera data and package repos have been restored '
+ 'to the previous working state.')
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
index 812a644..c60c0f5 100644
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
@@ -44,8 +44,7 @@ class Zeppelin(object):
def install(self):
'''
- Perform initial one-time setup, workaround upstream bugs, and
- trigger puppet.
+ Perform initial one-time setup and trigger puppet.
'''
# Dirs are handled by the bigtop deb, so no need to call out to
# dist_config to do that work. However, we want to adjust the
@@ -73,36 +72,8 @@ class Zeppelin(object):
self._add_override('spark::common::event_log_dir', events_log_dir)
self._add_override('spark::common::history_log_dir', events_log_dir)
- ##########
- # BUG: BIGTOP-2742
- # Default zeppelin init script looks for the literal '$(hostname)'
- # string. Symlink it so it exists before the apt install from puppet
- # tries to start the service.
- import subprocess
- host = subprocess.check_output(['hostname']).decode('utf8').strip()
- zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host)
- utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin')
- utils.run_as('root', 'ln', '-sf',
- zepp_pid,
- '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid')
- ##########
-
self.trigger_bigtop()
- ##########
- # BUG: BIGTOP-2742
- # Puppet apply will call systemctl daemon-reload, which removes the
- # symlink we just created. Now that the bits are on disk, update the
- # init script $(hostname) that caused this mess to begin with.
- zepp_init_script = '/etc/init.d/zeppelin'
- utils.re_edit_in_place(zepp_init_script, {
- r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid),
- })
- utils.run_as('root', 'systemctl', 'daemon-reload')
- self.restart()
- self.wait_for_api(30)
- ##########
-
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
@@ -124,8 +95,43 @@ class Zeppelin(object):
overrides=overrides,
)
- bigtop.trigger_puppet()
- self.wait_for_api(30)
+ # NB: during an upgrade, we configure the site.yaml, but do not
+ # trigger puppet. The user must do that with the 'reinstall' action.
+ if unitdata.kv().get('zeppelin.version.repo', False):
+ hookenv.log("An upgrade is available and the site.yaml has been "
+ "configured. Run the 'reinstall' action to continue.",
+ level=hookenv.INFO)
+ else:
+ ####################################################################
+ # BUG: BIGTOP-2742
+ # Default zeppelin init script looks for the literal '$(hostname)'
+ # string. Symlink it so it exists before the apt install from puppet
+ # tries to start the service.
+ import subprocess
+ host = subprocess.check_output(['hostname']).decode('utf8').strip()
+ zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host)
+ utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin')
+ utils.run_as('root', 'ln', '-sf',
+ zepp_pid,
+ '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid')
+ ####################################################################
+
+ bigtop.trigger_puppet()
+ self.wait_for_api(30)
+
+ ####################################################################
+ # BUG: BIGTOP-2742
+ # Puppet apply will call systemctl daemon-reload, which removes the
+ # symlink we just created. Now that the bits are on disk, update the
+ # init script $(hostname) that caused this mess to begin with.
+ zepp_init_script = '/etc/init.d/zeppelin'
+ utils.re_edit_in_place(zepp_init_script, {
+ r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid),
+ })
+ utils.run_as('root', 'systemctl', 'daemon-reload')
+ self.restart()
+ self.wait_for_api(30)
+ ####################################################################
def reconfigure_zeppelin(self):
'''
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py
index 7c9ca06..d68eccc 100644
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py
@@ -15,10 +15,9 @@
import hashlib
-from charms.reactive import when, when_not
-from charms.reactive import is_state, set_state, remove_state
-from charmhelpers.core import hookenv
-from charms.layer.apache_bigtop_base import get_package_version
+from charms.reactive import is_state, remove_state, set_state, when, when_not
+from charmhelpers.core import hookenv, unitdata
+from charms.layer.apache_bigtop_base import Bigtop, get_package_version
from charms.layer.bigtop_zeppelin import Zeppelin
from charms.reactive.helpers import data_changed
@@ -58,13 +57,19 @@ def update_status():
elif spark_ready:
ready_apps.append('spark')
- # Set appropriate status based on the apps we checked above
- if waiting_apps:
- hookenv.status_set('waiting',
- 'waiting for: {}'.format(' & '.join(waiting_apps)))
+ # Set appropriate status
+ repo_ver = unitdata.kv().get('zeppelin.version.repo', False)
+ if repo_ver:
+ # Pending upgrade takes precedent over other status messages
+ msg = "install version {} with the 'reinstall' action".format(repo_ver)
+ hookenv.status_set('active', msg)
+ elif waiting_apps:
+ # Waiting takes precedent over active status messages
+ msg = "waiting for: {}".format(' & '.join(waiting_apps))
+ hookenv.status_set('waiting', msg)
elif ready_apps:
- hookenv.status_set('active',
- 'ready with: {}'.format(' & '.join(ready_apps)))
+ msg = "ready with: {}".format(' & '.join(ready_apps))
+ hookenv.status_set('active', msg)
else:
hookenv.status_set('active', 'ready')
@@ -83,6 +88,27 @@ def initial_setup():
hookenv.application_version_set(zeppelin_version)
+@when('zeppelin.installed', 'bigtop.version.changed')
+def check_repo_version():
+ """
+ Configure a bigtop site.yaml if a new version of zeppelin is available.
+
+ This method will set unitdata if a different version of zeppelin is
+ available in the newly configured bigtop repo. This unitdata allows us to
+ configure site.yaml while gating the actual puppet apply. The user must do
+ the puppet apply by calling the 'reinstall' action.
+ """
+ repo_ver = Bigtop().check_bigtop_repo_package('zeppelin')
+ if repo_ver:
+ unitdata.kv().set('zeppelin.version.repo', repo_ver)
+ unitdata.kv().flush(True)
+ zeppelin = Zeppelin()
+ zeppelin.trigger_bigtop()
+ else:
+ unitdata.kv().unset('zeppelin.version.repo')
+ update_status()
+
+
@when('zeppelin.installed', 'hadoop.ready')
@when_not('zeppelin.hadoop.configured')
def configure_hadoop(hadoop):
@@ -170,7 +196,7 @@ def unconfigure_spark():
update_status()
-@when('zeppelin.started', 'client.notebook.registered')
+@when('zeppelin.installed', 'client.notebook.registered')
def register_notebook(client):
zeppelin = Zeppelin()
for notebook in client.unregistered_notebooks():
@@ -181,7 +207,7 @@ def register_notebook(client):
client.reject_notebook(notebook)
-@when('zeppelin.started', 'client.notebook.removed')
+@when('zeppelin.installed', 'client.notebook.removed')
def remove_notebook(client):
zeppelin = Zeppelin()
for notebook in client.unremoved_notebooks():
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py
index 0887cbe..b1ec990 100755
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/02-zeppelin-smoke.py
@@ -20,7 +20,7 @@ import re
import unittest
-class TestDeploy(unittest.TestCase):
+class TestSmoke(unittest.TestCase):
"""
Smoke test for Apache Bigtop Zeppelin.
"""
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
index 56f5f96..b7cb918 100755
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
@@ -20,7 +20,7 @@ import re
import unittest
-class TestDeploy(unittest.TestCase):
+class TestSmokeSpark(unittest.TestCase):
"""
Smoke test for Apache Bigtop Zeppelin using remote Spark resources.
"""
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ad7f6bd/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py
new file mode 100755
index 0000000..dac6190
--- /dev/null
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/04-zeppelin-config.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import amulet
+import re
+import unittest
+
+
+class TestConfig(unittest.TestCase):
+ """
+ Test configuring Apache Zeppelin.
+ """
+ @classmethod
+ def setUpClass(cls):
+ cls.d = amulet.Deployment(series='xenial')
+ cls.d.add('zeppelin-test-config', charm='zeppelin',
+ constraints={'mem': '7G'})
+ cls.d.setup(timeout=1800)
+ cls.d.sentry.wait_for_messages({'zeppelin-test-config': re.compile('ready')},
+ timeout=1800)
+ cls.zeppelin = cls.d.sentry['zeppelin-test-config'][0]
+
+ def test_bigtop_upgrade(self):
+ """
+ Validate Zeppelin status is changed when upgrading zeppelin.
+ """
+ self.d.configure('zeppelin-test-config',
+ {'bigtop_version': 'master'})
+ self.d.sentry.wait_for_messages({'zeppelin-test-config': re.compile('reinstall|ready')},
+ timeout=900)
+
+
+if __name__ == '__main__':
+ unittest.main()