You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2018/01/05 23:04:28 UTC
[32/50] [abbrv] ambari git commit: AMBARI-22655 - Livy/Livy2 Unable
To Start Due to Address Already In Use (jonathanhurley)
AMBARI-22655 - Livy/Livy2 Unable To Start Due to Address Already In Use (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4e1a2bf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4e1a2bf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4e1a2bf
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: f4e1a2bff9b93f28ddc5fa958739a81fe3148aa5
Parents: 7749e65
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Dec 14 21:59:19 2017 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Fri Jan 5 17:54:16 2018 -0500
----------------------------------------------------------------------
.../SPARK/1.2.1/package/scripts/livy_service.py | 22 +++++++++------
.../SPARK/2.2.0/package/scripts/livy_service.py | 29 +++++++++-----------
.../2.0.0/package/scripts/livy2_service.py | 27 ++++++++----------
.../SPARK2/2.0.0/package/scripts/setup_livy2.py | 2 ++
.../HDP/2.0.6/properties/stack_packages.json | 8 +++++-
.../HDP/3.0/properties/stack_packages.json | 8 +++++-
.../python/stacks/2.5/SPARK/test_spark_livy.py | 3 +-
.../stacks/2.6/SPARK2/test_spark_livy2.py | 5 +++-
8 files changed, 60 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
index cce2148..add5a77 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
@@ -21,28 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy_server_pid_file}` >/dev/null 2>&1')
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat {livy_server_pid_file}"),
+ user=params.livy_user, is_checked_call=False)[1]
+
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy_server_start}'),
user=params.livy_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test)
+ not_if=process_id_exists_command
+ )
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy_server_pid_file} || cat {livy_server_pid_file}"), user=params.livy_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy_server_stop}'),
user=params.livy_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
+
File(params.livy_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
index 273b056..add5a77 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/livy_service.py
@@ -21,35 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy_server_pid_file}` >/dev/null 2>&1')
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat {livy_server_pid_file}"),
+ user=params.livy_user, is_checked_call=False)[1]
+
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy_server_start}'),
user=params.livy_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test
- )
+ not_if=process_id_exists_command
+ )
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy_server_pid_file} || cat {livy_server_pid_file}"), user=params.livy_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy_server_stop}'),
user=params.livy_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
- File(params.livy_server_pid_file,
- action="delete"
- )
-
-
-
+ File(params.livy_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
index dfadd84..0180a31 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
@@ -21,35 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy2_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
- 'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat {livy2_server_pid_file}` >/dev/null 2>&1')
+ # use the livy2 user to get the PID (it is protected on non-root systems)
+ livy2_server_pid = get_user_call_output.get_user_call_output(format("cat {livy2_server_pid_file}"),
+ user=params.livy2_user, is_checked_call=False)[1]
+
+ livy2_server_pid = livy2_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p {livy2_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy2_server_start}'),
user=params.livy2_user,
environment={'JAVA_HOME': params.java_home},
- not_if=livyserver_no_op_test
+ not_if=process_id_exists_command
)
elif action == 'stop':
- pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy2_server_pid_file} || cat {livy2_server_pid_file}"), user=params.livy2_user)[1]
- pid = pid.replace("\n", " ")
Execute(format('{livy2_server_stop}'),
user=params.livy2_user,
- only_if=livyserver_no_op_test,
+ only_if=process_id_exists_command,
timeout=10,
- on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E kill -9 {pid}"),
+ on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy2_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
- File(params.livy2_server_pid_file,
- action="delete"
- )
-
-
-
+ File(params.livy2_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
index c428488..59ec190 100644
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py
@@ -29,8 +29,10 @@ def setup_livy(env, type, upgrade_type = None, action = None):
owner=params.livy2_user,
group=params.user_group,
mode=0775,
+ cd_access = 'a',
create_parents = True
)
+
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index 01a30f3..6a0eadd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -1067,6 +1067,12 @@
"current_dir": "{0}/current/livy-client/conf"
}
],
+ "livy2": [
+ {
+ "conf_dir": "/etc/livy2/conf",
+ "current_dir": "{0}/current/livy2-client/conf"
+ }
+ ],
"mahout": [
{
"conf_dir": "/etc/mahout/conf",
@@ -1241,7 +1247,7 @@
"packages": ["spark", "livy"]
},
"SPARK2": {
- "packages": ["spark2", "livy"]
+ "packages": ["spark2", "livy2"]
},
"SQOOP": {
"packages": ["sqoop"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
index 28efbfa..63fcf4b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_packages.json
@@ -944,6 +944,12 @@
"current_dir": "{0}/current/livy-client/conf"
}
],
+ "livy2": [
+ {
+ "conf_dir": "/etc/livy2/conf",
+ "current_dir": "{0}/current/livy2-client/conf"
+ }
+ ],
"mahout": [
{
"conf_dir": "/etc/mahout/conf",
@@ -1118,7 +1124,7 @@
"packages": ["spark", "livy"]
},
"SPARK2": {
- "packages": ["spark2", "livy"]
+ "packages": ["spark2", "livy2"]
},
"SQOOP": {
"packages": ["sqoop"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
index 4f1fb17..e37c6fa 100644
--- a/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
+++ b/ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py
@@ -25,6 +25,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597"))
+@patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0,'12345','')))
class TestSparkClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.1/package"
STACK_VERSION = "2.5"
@@ -146,6 +147,6 @@ class TestSparkClient(RMFTestCase):
)
self.assertResourceCalled('Execute', '/usr/hdp/current/livy-server/bin/livy-server start',
environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'},
- not_if = 'ls /var/run/livy/livy-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy/livy-livy-server.pid` >/dev/null 2>&1',
+ not_if = 'ls /var/run/livy/livy-livy-server.pid >/dev/null 2>&1 && ps -p 12345 >/dev/null 2>&1',
user = 'livy'
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4e1a2bf/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
index 60e7fd2..f0a173b 100644
--- a/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
+++ b/ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py
@@ -25,6 +25,7 @@ from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_stack_version", new=MagicMock(return_value="2.5.0.0-1597"))
+@patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", new=MagicMock(return_value=(0,'12345','')))
class TestSparkClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK2/2.0.0/package"
STACK_VERSION = "2.6"
@@ -45,12 +46,14 @@ class TestSparkClient(RMFTestCase):
owner = 'livy',
group = 'hadoop',
create_parents = True,
+ cd_access = 'a',
mode = 0775
)
self.assertResourceCalled('Directory', '/var/log/livy2',
owner = 'livy',
group = 'hadoop',
create_parents = True,
+ cd_access = 'a',
mode = 0775
)
self.assertResourceCalled('HdfsResource', '/user/livy',
@@ -146,6 +149,6 @@ class TestSparkClient(RMFTestCase):
)
self.assertResourceCalled('Execute', '/usr/hdp/current/livy2-server/bin/livy-server start',
environment = {'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'},
- not_if = 'ls /var/run/livy2/livy-livy-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/livy2/livy-livy-server.pid` >/dev/null 2>&1',
+ not_if = 'ls /var/run/livy2/livy-livy-server.pid >/dev/null 2>&1 && ps -p 12345 >/dev/null 2>&1',
user = 'livy'
)