You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/13 22:13:30 UTC
[22/50] [abbrv] ambari git commit: AMBARI-19577. Add Livy session
recovery configurations in Ambari (Saisai Shao via smohanty)
AMBARI-19577. Add Livy session recovery configurations in Ambari (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0ba7fbe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0ba7fbe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0ba7fbe
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: a0ba7fbe7f80332bcff1cab9ea7dbe27ca59a003
Parents: eecb5a3
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Feb 10 08:36:58 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Feb 10 08:42:42 2017 -0800
----------------------------------------------------------------------
.../SPARK/1.2.1/package/scripts/params.py | 1 +
.../SPARK/1.2.1/package/scripts/setup_livy.py | 8 +++++
.../SPARK2/2.0.0/package/scripts/params.py | 1 +
.../SPARK2/2.0.0/package/scripts/setup_livy2.py | 8 +++++
.../services/SPARK/configuration/livy-conf.xml | 24 +++++++++++++++
.../SPARK2/configuration/livy2-conf.xml | 24 +++++++++++++++
.../python/stacks/2.5/SPARK/test_spark_livy.py | 31 ++++++++++++++++++++
.../stacks/2.6/SPARK2/test_spark_livy2.py | 31 ++++++++++++++++++++
8 files changed, 128 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
index 82cd0b2..6a59caf 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
@@ -222,6 +222,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stac
user_group = status_params.user_group
livy_hdfs_user_dir = format("/user/{livy_user}")
livy_server_pid_file = status_params.livy_server_pid_file
+ livy_recovery_dir = default("/configurations/livy-conf/livy.server.recovery.state-store.url", "/livy-recovery")
livy_server_start = format("{livy_home}/bin/livy-server start")
livy_server_stop = format("{livy_home}/bin/livy-server stop")
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
index 07b704f..32615c3 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py
@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
)
params.HdfsResource(None, action="execute")
+ params.HdfsResource(params.livy_recovery_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.livy_user,
+ mode=0775
+ )
+ params.HdfsResource(None, action="execute")
+
# create livy-env.sh in etc/conf dir
File(os.path.join(params.livy_conf, 'livy-env.sh'),
owner=params.livy_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
index c19f88c..b6889e4 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py
@@ -207,6 +207,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, sta
user_group = status_params.user_group
livy2_hdfs_user_dir = format("/user/{livy2_user}")
livy2_server_pid_file = status_params.livy2_server_pid_file
+ livy2_recovery_dir = default("/configurations/livy2-conf/livy.server.recovery.state-store.url", "/livy2-recovery")
livy2_server_start = format("{livy2_home}/bin/livy-server start")
livy2_server_stop = format("{livy2_home}/bin/livy-server stop")
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
index f2178f7..2e92509 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
)
params.HdfsResource(None, action="execute")
+ params.HdfsResource(params.livy2_recovery_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.livy2_user,
+ mode=0775
+ )
+ params.HdfsResource(None, action="execute")
+
# create livy-env.sh in etc/conf dir
File(os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
index 23583c0..28eeab5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml
@@ -78,4 +78,28 @@
</description>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>livy.server.recovery.mode</name>
+ <value>recovery</value>
+ <description>
+ Recovery mode for livy, either be "off" or "recovery".
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store</name>
+ <value>filesystem</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store.url</name>
+ <value>/livy-recovery</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
index 231ef70..c8a65bd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml
@@ -85,6 +85,30 @@
Whether to enable HiveContext in livy interpreter
</description>
<on-ambari-upgrade add="false"/>
+ <property>
+ </property>
+ <name>livy.server.recovery.mode</name>
+ <value>recovery</value>
+ <description>
+ Recovery mode for livy, either be "off" or "recovery".
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store</name>
+ <value>filesystem</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>livy.server.recovery.state-store.url</name>
+ <value>/livy2-recovery</value>
+ <description>
+ Where Livy should store state for recovery.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
index b56474a..b9199c7 100644
--- a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
+++ b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
dfs_type = '',
hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
)
+ self.assertResourceCalled('HdfsResource', '/livy-recovery',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'livy',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ mode = 0775,
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
self.assertResourceCalled('File', '/usr/hdp/current/livy-server/conf/livy-env.sh',
content = InlineTemplate(self.getConfig()['configurations']['livy-env']['content']),
owner = 'livy',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a0ba7fbe/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
index 6244d87..75aec84 100644
--- a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
+++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
dfs_type = '',
hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
)
+ self.assertResourceCalled('HdfsResource', '/livy2-recovery',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'livy',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ mode = 0775,
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy-env.sh',
content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']),
owner = 'livy',