You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/03/06 18:44:29 UTC
[16/25] ambari git commit: AMBARI-20200: hive-site.xml, hbase-site.xml,
etc. are not found in class path for Zeppelin (Prabhjyot Singh via
r-kamath)
AMBARI-20200: hive-site.xml, hbase-site.xml, etc. are not found in class path for Zeppelin (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6025d413
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6025d413
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6025d413
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 6025d413da204a4cf5cf69fe90a413b905ac9344
Parents: 109bba1
Author: Renjith Kamath <re...@gmail.com>
Authored: Mon Mar 6 15:56:19 2017 +0530
Committer: Renjith Kamath <re...@gmail.com>
Committed: Mon Mar 6 15:56:19 2017 +0530
----------------------------------------------------------------------
.../0.6.0.2.5/configuration/zeppelin-env.xml | 2 +-
.../0.6.0.2.5/package/scripts/master.py | 36 +++++++++++++++-----
.../0.6.0.2.5/package/scripts/params.py | 8 +++++
.../stacks/2.5/ZEPPELIN/test_zeppelin_master.py | 26 +++++++-------
4 files changed, 49 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
index 677158c..bdef319 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
@@ -89,7 +89,7 @@ export ZEPPELIN_PID_DIR={{zeppelin_pid_dir}}
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise
-
+export ZEPPELIN_INTP_CLASSPATH_OVERRIDES="{{external_dependency_conf}}"
#### Spark interpreter configuration ####
## Use provided spark installation ##
http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 8a1fad6..fc24918 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -106,6 +106,17 @@ class Master(Script):
mode=0755
)
+ def create_zeppelin_hdfs_conf_dir(self, env):
+ import params
+ env.set_params(params)
+ Directory([params.external_dependency_conf],
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ cd_access="a",
+ create_parents=True,
+ mode=0755
+ )
+
def chown_zeppelin_pid_dir(self, env):
import params
env.set_params(params)
@@ -150,17 +161,26 @@ class Master(Script):
File(format("{params.conf_dir}/log4j.properties"), content=params.log4j_properties_content,
owner=params.zeppelin_user, group=params.zeppelin_group)
+ self.create_zeppelin_hdfs_conf_dir(env)
# copy hive-site.xml only if Spark 1.x is installed
- if 'spark-defaults' in params.config['configurations'] and \
- os.path.exists("/etc/spark/conf/hive-site.xml"):
- File(format("{params.conf_dir}/hive-site.xml"), content=StaticFile("/etc/spark/conf/hive-site.xml"),
- owner=params.zeppelin_user, group=params.zeppelin_group)
+ if 'spark-defaults' in params.config['configurations'] and params.is_hive_installed:
+ XmlConfig("hive-site.xml",
+ conf_dir=params.external_dependency_conf,
+ configurations=params.spark_hive_properties,
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ mode=0644)
- if len(params.hbase_master_hosts) > 0 and \
- os.path.exists("/etc/hbase/conf/hbase-site.xml"):
+ if len(params.hbase_master_hosts) > 0 and params.is_hbase_installed:
# copy hbase-site.xml
- File(format("{params.conf_dir}/hbase-site.xml"), content=StaticFile("/etc/hbase/conf/hbase-site.xml"),
- owner=params.zeppelin_user, group=params.zeppelin_group)
+ XmlConfig("hbase-site.xml",
+ conf_dir=params.external_dependency_conf,
+ configurations=params.config['configurations']['hbase-site'],
+ configuration_attributes=params.config['configuration_attributes']['hbase-site'],
+ owner=params.zeppelin_user,
+ group=params.zeppelin_group,
+ mode=0644
+ )
def stop(self, env, upgrade_type=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index 16a2782..9fd105b 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -95,6 +95,7 @@ zeppelin_hdfs_user_dir = format("/user/{zeppelin_user}")
zeppelin_dir = os.path.join(*[install_dir, zeppelin_dirname])
conf_dir = "/etc/zeppelin/conf"
+external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf"
notebook_dir = os.path.join(*[install_dir, zeppelin_dirname, 'notebook'])
# zeppelin-env.sh
@@ -121,7 +122,12 @@ hive_metastore_port = None
hive_server_port = None
hive_zookeeper_quorum = None
hive_server2_support_dynamic_service_discovery = None
+is_hive_installed = False
if 'hive_server_host' in master_configs and len(master_configs['hive_server_host']) != 0:
+ is_hive_installed = True
+ spark_hive_properties = {
+ 'hive.metastore.uris': config['configurations']['hive-site']['hive.metastore.uris']
+ }
hive_server_host = str(master_configs['hive_server_host'][0])
hive_metastore_host = str(master_configs['hive_metastore_host'][0])
hive_metastore_port = str(
@@ -133,7 +139,9 @@ if 'hive_server_host' in master_configs and len(master_configs['hive_server_host
# detect hbase details if installed
zookeeper_znode_parent = None
hbase_zookeeper_quorum = None
+is_hbase_installed = False
if 'hbase_master_hosts' in master_configs and 'hbase-site' in config['configurations']:
+ is_hbase_installed = True
zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent']
hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum']
http://git-wip-us.apache.org/repos/asf/ambari/blob/6025d413/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
index bd0f5ef..60c18c9 100644
--- a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
+++ b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
@@ -75,10 +75,12 @@ class TestZeppelinMaster(RMFTestCase):
content = u'log4j.rootLogger = INFO, dailyfile',
group = u'zeppelin',
)
- self.assertResourceCalled('File', '/etc/zeppelin/conf/hive-site.xml',
+ self.assertResourceCalled('Directory', '/etc/zeppelin/conf/external-dependency-conf',
owner = 'zeppelin',
- content = StaticFile('/etc/spark/conf/hive-site.xml'),
group = 'zeppelin',
+ create_parents = True,
+ mode = 0755,
+ cd_access = 'a',
)
def assert_configure_secured(self):
@@ -125,15 +127,15 @@ class TestZeppelinMaster(RMFTestCase):
content = u'log4j.rootLogger = INFO, dailyfile',
group = u'zeppelin',
)
- self.assertResourceCalled('File', '/etc/zeppelin/conf/hive-site.xml',
+ self.assertResourceCalled('Directory', '/etc/zeppelin/conf/external-dependency-conf',
owner = 'zeppelin',
- content = StaticFile('/etc/spark/conf/hive-site.xml'),
group = 'zeppelin',
+ create_parents = True,
+ mode = 0755,
+ cd_access = 'a',
)
- @patch('os.path.exists')
- def test_configure_default(self, os_path_exists_mock):
- os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml'
+ def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py",
classname = "Master",
command = "configure",
@@ -144,9 +146,7 @@ class TestZeppelinMaster(RMFTestCase):
self.assert_configure_default()
self.assertNoMoreResources()
- @patch('os.path.exists')
- def test_configure_secured(self, os_path_exists_mock):
- os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml'
+ def test_configure_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py",
classname = "Master",
command = "configure",
@@ -202,10 +202,8 @@ class TestZeppelinMaster(RMFTestCase):
user = 'zeppelin',
)
self.assertNoMoreResources()
-
- @patch('os.path.exists')
- def test_start_default(self, os_path_exists_mock):
- os_path_exists_mock.side_effect = lambda path: path == '/etc/spark/conf/hive-site.xml'
+
+ def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/master.py",
classname = "Master",
command = "start",