You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/03/30 04:02:19 UTC
ambari git commit: AMBARI-15114: Stack Featurize HBASE Service
(Juanjo Marron via jluniya)
Repository: ambari
Updated Branches:
refs/heads/trunk 5c20a2488 -> 954a7072b
AMBARI-15114: Stack Featurize HBASE Service (Juanjo Marron via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/954a7072
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/954a7072
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/954a7072
Branch: refs/heads/trunk
Commit: 954a7072b0d3d82e5f818794b0a192a743b9c76a
Parents: 5c20a24
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Tue Mar 29 19:02:06 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Tue Mar 29 19:02:06 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../libraries/functions/stack_features.py | 5 +++
.../0.96.0.2.0/package/scripts/hbase_client.py | 7 ++--
.../0.96.0.2.0/package/scripts/hbase_master.py | 3 +-
.../package/scripts/hbase_regionserver.py | 3 +-
.../0.96.0.2.0/package/scripts/params_linux.py | 36 +++++++++++---------
.../package/scripts/phoenix_queryserver.py | 9 +++--
.../package/scripts/phoenix_service.py | 2 +-
.../0.96.0.2.0/package/scripts/status_params.py | 11 ++++--
.../HBASE/0.96.0.2.0/package/scripts/upgrade.py | 5 +--
.../HDP/2.0.6/properties/stack_features.json | 5 +++
11 files changed, 58 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 7ec83b6..b6aa1b8 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -46,6 +46,7 @@ class StackFeature:
DATANODE_NON_ROOT = "datanode_non_root"
REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env"
RANGER = "ranger"
+ PHOENIX = "phoenix"
NFS = "nfs"
TEZ_FOR_SPARK = "tez_for_spark"
SPARK_16PLUS = "spark_16plus"
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index d79f743..e5fdd89 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -63,6 +63,11 @@ _DEFAULT_STACK_FEATURES = {
"min_version": "2.2.0.0"
},
{
+ "name": "phoenix",
+ "description": "Phoenix Service support",
+ "min_version": "2.3.0.0"
+ },
+ {
"name": "nfs",
"description": "NFS support",
"min_version": "2.3.0.0"
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py
index c31bbf6..b72e648 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_client.py
@@ -22,6 +22,8 @@ import sys
from resource_management import *
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
from hbase import hbase
from ambari_commons import OSCheck, OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
@@ -51,13 +53,14 @@ class HbaseClientWindows(HbaseClient):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class HbaseClientDefault(HbaseClient):
def get_stack_to_component(self):
- return {"HDP": "hbase-client"}
+ import params
+ return {params.stack_name: "hbase-client"}
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
- if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
conf_select.select(params.stack_name, "hbase", params.version)
stack_select.select("hbase-client", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py
index 9515f61..5833773 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py
@@ -71,7 +71,8 @@ class HbaseMasterWindows(HbaseMaster):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class HbaseMasterDefault(HbaseMaster):
def get_stack_to_component(self):
- return {"HDP": "hbase-master"}
+ import params
+ return {params.stack_name: "hbase-master"}
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
index c17d219..37dd0a1 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
@@ -69,7 +69,8 @@ class HbaseRegionServerWindows(HbaseRegionServer):
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
class HbaseRegionServerDefault(HbaseRegionServer):
def get_stack_to_component(self):
- return {"HDP": "hbase-regionserver"}
+ import params
+ return {params.stack_name: "hbase-regionserver"}
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index bc6fcde..31e1757 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -29,7 +29,8 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions import format
-from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions import is_empty
@@ -51,8 +52,9 @@ version = default("/commandParams/version", None)
component_directory = status_params.component_directory
etc_prefix_dir = "/etc/hbase"
-stack_version_unformatted = config['hostLevelParams']['stack_version']
-stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_version_unformatted = status_params.stack_version_unformatted
+stack_version_formatted = status_params.stack_version_formatted
+stack_root = status_params.stack_root
# hadoop default parameters
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
@@ -63,19 +65,19 @@ region_drainer = "/usr/lib/hbase/bin/draining_servers.rb"
hbase_cmd = "/usr/lib/hbase/bin/hbase"
hbase_max_direct_memory_size = None
-# hadoop parameters for 2.2+
-if Script.is_stack_greater_or_equal("2.2"):
- daemon_script = format('/usr/hdp/current/hbase-client/bin/hbase-daemon.sh')
- region_mover = format('/usr/hdp/current/hbase-client/bin/region_mover.rb')
- region_drainer = format('/usr/hdp/current/hbase-client/bin/draining_servers.rb')
- hbase_cmd = format('/usr/hdp/current/hbase-client/bin/hbase')
+# hadoop parameters for stacks supporting rolling_upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ daemon_script = format('{stack_root}/current/hbase-client/bin/hbase-daemon.sh')
+ region_mover = format('{stack_root}/current/hbase-client/bin/region_mover.rb')
+ region_drainer = format('{stack_root}/current/hbase-client/bin/draining_servers.rb')
+ hbase_cmd = format('{stack_root}/current/hbase-client/bin/hbase')
hbase_max_direct_memory_size = default('configurations/hbase-env/hbase_max_direct_memory_size', None)
- daemon_script=format("/usr/hdp/current/{component_directory}/bin/hbase-daemon.sh")
- region_mover = format("/usr/hdp/current/{component_directory}/bin/region_mover.rb")
- region_drainer = format("/usr/hdp/current/{component_directory}/bin/draining_servers.rb")
- hbase_cmd = format("/usr/hdp/current/{component_directory}/bin/hbase")
+ daemon_script=format("{stack_root}/current/{component_directory}/bin/hbase-daemon.sh")
+ region_mover = format("{stack_root}/current/{component_directory}/bin/region_mover.rb")
+ region_drainer = format("{stack_root}/current/{component_directory}/bin/draining_servers.rb")
+ hbase_cmd = format("{stack_root}/current/{component_directory}/bin/hbase")
hbase_conf_dir = status_params.hbase_conf_dir
@@ -85,7 +87,7 @@ hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_l
hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000")
# no symlink for phoenix-server at this point
-phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py'
+phx_daemon_script = format('{stack_root}/current/phoenix-server/bin/queryserver.py')
hbase_excluded_hosts = config['commandParams']['excluded_hosts']
hbase_drain_only = default("/commandParams/mark_draining_only",False)
@@ -313,7 +315,7 @@ if has_ranger_admin:
downloaded_custom_connector = format("{exec_tmp_dir}/{jdbc_jar_name}")
driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}")
- driver_curl_target = format("/usr/hdp/current/{component_directory}/lib/{jdbc_jar_name}")
+ driver_curl_target = format("{stack_root}/current/{component_directory}/lib/{jdbc_jar_name}")
hbase_ranger_plugin_config = {
'username': repo_config_username,
@@ -353,11 +355,11 @@ if security_enabled:
hbase_coprocessor_master_classes = "org.apache.hadoop.hbase.security.access.AccessController"
hbase_coprocessor_regionserver_classes = "org.apache.hadoop.hbase.security.access.AccessController"
hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.hadoop.hbase.security.access.AccessController"
- elif xml_configurations_supported: # HDP stack 2.3+ ranger plugin enabled
+ elif xml_configurations_supported: # stack version ranger plugin enabled
hbase_coprocessor_master_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor "
hbase_coprocessor_regionserver_classes = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
- else: # HDP Stack 2.2 and less / ranger plugin enabled
+ else:
hbase_coprocessor_master_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
hbase_coprocessor_regionserver_classes = "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
hbase_coprocessor_region_classes = "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor"
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py
index 87e4899..7d639ec 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py
@@ -19,11 +19,13 @@ limitations under the License.
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
from phoenix_service import phoenix_service
from hbase import hbase
-# Note: Phoenix Query Server is only applicable to HDP-2.3 and above.
+# Note: Phoenix Query Server is only applicable to stack version supporting Phoenix.
class PhoenixQueryServer(Script):
def install(self, env):
@@ -33,7 +35,8 @@ class PhoenixQueryServer(Script):
def get_stack_to_component(self):
- return {"HDP": "phoenix-server"}
+ import params
+ return {params.stack_name: "phoenix-server"}
def configure(self, env):
@@ -59,7 +62,7 @@ class PhoenixQueryServer(Script):
import params
env.set_params(params)
- if Script.is_stack_greater_or_equal("2.3"):
+ if params.stack_version_formatted and check_stack_feature(StackFeature.PHOENIX, params.stack_version_formatted):
# phoenix uses hbase configs
conf_select.select(params.stack_name, "hbase", params.version)
stack_select.select("phoenix-server", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py
index e155cec..d1e2bbd 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_service.py
@@ -22,7 +22,7 @@ from resource_management.core.resources.system import Execute
from resource_management.core.resources.system import File
from resource_management.libraries.functions import check_process_status, format
-# Note: Phoenix Query Server is only applicable to HDP-2.3 and above.
+# Note: Phoenix Query Server is only applicable to phoenix version stacks and above.
def phoenix_service(action = 'start'): # 'start', 'stop', 'status'
# Note: params/status_params should already be imported before calling phoenix_service()
pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
index 3ec84ee..8b60292 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
@@ -21,6 +21,9 @@ from ambari_commons.os_check import OSCheck
from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
@@ -48,8 +51,12 @@ else:
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
+
+ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+ stack_version_formatted = format_stack_version(stack_version_unformatted)
+ stack_root = Script.get_stack_root()
hbase_conf_dir = "/etc/hbase/conf"
limits_conf_dir = "/etc/security/limits.d"
- if Script.is_stack_greater_or_equal("2.2"):
- hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf")
\ No newline at end of file
+ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ hbase_conf_dir = format("{stack_root}/current/{component_directory}/conf")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py
index 92c0f70..01a8156 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/upgrade.py
@@ -23,13 +23,14 @@ from resource_management.core.resources.system import Execute
from resource_management.core import shell
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.decorator import retry
def prestart(env, stack_component):
import params
- if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
conf_select.select(params.stack_name, "hbase", params.version)
stack_select.select(stack_component, params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/954a7072/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index d38985e..0234bba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -37,6 +37,11 @@
"min_version": "2.2.0.0"
},
{
+ "name": "phoenix",
+ "description": "Phoenix Service support",
+ "min_version": "2.3.0.0"
+ },
+ {
"name": "nfs",
"description": "NFS support",
"min_version": "2.3.0.0"