You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2016/03/23 07:38:26 UTC
ambari git commit: AMBARI-14435: Stack Featurization for ZOOKEEPER
service (Juanjo Marron via jluniya)
Repository: ambari
Updated Branches:
refs/heads/trunk cb98bcb91 -> 215a61df1
AMBARI-14435: Stack Featurization for ZOOKEEPER service (Juanjo Marron via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/215a61df
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/215a61df
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/215a61df
Branch: refs/heads/trunk
Commit: 215a61df16d93b781e0cfa5c424d847e2c735efb
Parents: cb98bcb
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Tue Mar 22 23:38:17 2016 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Tue Mar 22 23:38:17 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/constants.py | 14 +++-
.../libraries/functions/stack_features.py | 67 ++++++++++++++++++++
.../3.4.5.2.0/package/scripts/params_linux.py | 16 +++--
.../3.4.5.2.0/package/scripts/status_params.py | 11 +++-
.../3.4.5.2.0/package/scripts/zookeeper.py | 6 +-
.../package/scripts/zookeeper_client.py | 4 +-
.../package/scripts/zookeeper_server.py | 8 ++-
.../package/scripts/zookeeper_service.py | 6 +-
.../HDP/2.0.6/configuration/cluster-env.xml | 11 ++++
.../HDP/2.0.6/properties/stack_features.json | 15 +++++
10 files changed, 140 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 0adaef9..262213b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -18,7 +18,7 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
-__all__ = ["Direction", "SafeMode"]
+__all__ = ["Direction", "SafeMode", "StackFeature"]
class Direction:
"""
@@ -34,3 +34,15 @@ class SafeMode:
ON = "ON"
OFF = "OFF"
UNKNOWN = "UNKNOWN"
+
+class StackFeature:
+ """
+ Stack Feature supported
+ """
+ SNAPPY = "snappy"
+ EXPRESS_UPGRADE = "express_upgrade"
+ ROLLING_UPGRADE = "rolling_upgrade"
+ CONFIG_VERSIONING = "config_versioning"
+ RANGER = "ranger"
+ NFS = "nfs"
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
new file mode 100644
index 0000000..ac38218
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+# simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import ambari_simplejson as json
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.default import default
+
+_DEFAULT_STACK_FEATURES = {
+ "stack_features": [
+ { "name": "snappy", "description" : "Snappy compressor/decompressor support" , "min_version" : "2.0.0.0" , "max_version" : "2.2.0.0" }
+ ,
+ { "name": "express_upgrade", "description" : "Express upgrade support", "min_version" : "2.1.0.0" }
+ ,
+ { "name": "rolling_upgrade", "description" : "Rolling upgrade support", "min_version" : "2.2.0.0" }
+ ,
+ { "name": "config_versioning", "description" : "Configurable versions support", "min_version" : "2.3.0.0" }
+ ,
+ { "name": "ranger", "description" : "Ranger Service support", "min_version" : "2.2.0.0" }
+ ,
+ { "name": "nfs", "description" : "NFS support", "min_version" : "2.3.0.0" }
+ ]
+}
+
+def check_stack_feature(stack_feature, stack_version):
+ """
+ Given a stack_feature and a specific stack_version, it validates that the feature is supported by the stack_version.
+ :param stack_feature: Feature name to check if it is supported by the stack. For example: "rolling_upgrade"
+ :param stack_version: Version of the stack
+ :return: Will return True if successful, otherwise, False.
+ """
+ stack_features_config = default("/configurations/cluster-env/stack_features", None)
+ data = _DEFAULT_STACK_FEATURES
+
+ if stack_features_config:
+ data = json.loads(stack_features_config)
+
+ for feature in data["stack_features"]:
+ if feature["name"] == stack_feature:
+ if "min_version" in feature:
+ min_version = feature["min_version"]
+ if Script.is_stack_less_than(min_version):
+ return False
+ if "max_version" in feature:
+ max_version = feature["max_version"]
+ if Script.is_stack_greater_or_equal(max_version):
+ return False
+ return True
+
+ return False
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py
index 67d0776..5a1b129 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py
@@ -24,15 +24,17 @@ import os
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.script.script import Script
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_version_formatted = status_params.stack_version_formatted
+stack_root = status_params.stack_root
stack_name = default("/hostLevelParams/stack_name", None)
current_version = default("/hostLevelParams/current_version", None)
@@ -48,11 +50,11 @@ zk_cli_shell = "/usr/lib/zookeeper/bin/zkCli.sh"
config_dir = "/etc/zookeeper/conf"
zk_smoke_out = os.path.join(tmp_dir, "zkSmoke.out")
-# hadoop parameters for 2.2+
-if Script.is_stack_greater_or_equal("2.2"):
- zk_home = format("/usr/hdp/current/{component_directory}")
- zk_bin = format("/usr/hdp/current/{component_directory}/bin")
- zk_cli_shell = format("/usr/hdp/current/{component_directory}/bin/zkCli.sh")
+# hadoop parameters for stacks that support rolling_upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ zk_home = format("{stack_root}/current/{component_directory}")
+ zk_bin = format("{stack_root}/current/{component_directory}/bin")
+ zk_cli_shell = format("{stack_root}/current/{component_directory}/bin/zkCli.sh")
config_dir = status_params.config_dir
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
index 9ae8440..569385d 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py
@@ -20,6 +20,9 @@ limitations under the License.
from ambari_commons import OSCheck
from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
@@ -46,7 +49,11 @@ else:
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
zk_user = config['configurations']['zookeeper-env']['zk_user']
+
+ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+ stack_version_formatted = format_stack_version(stack_version_unformatted)
+ stack_root = Script.get_stack_root()
config_dir = "/etc/zookeeper/conf"
- if Script.is_stack_greater_or_equal("2.2"):
- config_dir = format("/usr/hdp/current/{component_directory}/conf")
+ if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ config_dir = format("{stack_root}/current/{component_directory}/conf")
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py
index 5712ce4..e0ba54b 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py
@@ -24,7 +24,9 @@ import sys
from resource_management import *
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -35,8 +37,8 @@ def zookeeper(type = None, upgrade_type=None):
if type == 'server':
# This path may be missing after Ambari upgrade. We need to create it. We need to do this before any configs will
# be applied.
- if upgrade_type is None and not os.path.exists("/usr/hdp/current/zookeeper-server") and params.current_version\
- and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+ if upgrade_type is None and not os.path.exists(os.path.join(params.stack_root,"/current/zookeeper-server")) and params.current_version\
+ and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
conf_select.select(params.stack_name, "zookeeper", params.current_version)
stack_select.select("zookeeper-server", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py
index 25ace24..ed6749a 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py
@@ -23,8 +23,10 @@ import sys
from resource_management import *
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.version import compare_versions, format_stack_version
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.stack_features import check_stack_feature
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -65,7 +67,7 @@ class ZookeeperClientLinux(ZookeeperClient):
import params
env.set_params(params)
- if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
conf_select.select(params.stack_name, "zookeeper", params.version)
stack_select.select("zookeeper-client", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
index d5b6898..54db974 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py
@@ -25,7 +25,9 @@ from resource_management.libraries.script.script import Script
from resource_management.libraries.functions import get_unique_id_and_date
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.security_commons import build_expectations, \
cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_JAAS_CONF
@@ -73,8 +75,8 @@ class ZookeeperServerLinux(ZookeeperServer):
Logger.info("Executing Stack Upgrade pre-restart")
import params
env.set_params(params)
-
- if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
conf_select.select(params.stack_name, "zookeeper", params.version)
stack_select.select("zookeeper-server", params.version)
@@ -114,7 +116,7 @@ class ZookeeperServerLinux(ZookeeperServer):
env.set_params(status_params)
if status_params.security_enabled:
- # Expect the following files to be available in status_params.config_dir:
+ # Expect the following files to be available in params.config_dir:
# zookeeper_jaas.conf
# zookeeper_client_jaas.conf
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py
index 6fe0772..d645997 100644
--- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py
+++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py
@@ -25,15 +25,17 @@ from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def zookeeper_service(action='start', upgrade_type=None):
import params
# This path may be missing after Ambari upgrade. We need to create it.
- if upgrade_type is None and not os.path.exists("/usr/hdp/current/zookeeper-server") and params.current_version \
- and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+ if upgrade_type is None and not os.path.exists(os.path.join(params.stack_root,"/current/zookeeper-server")) and params.current_version \
+ and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
conf_select.select(params.stack_name, "zookeeper", params.current_version)
stack_select.select("zookeeper-server", params.version)
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
index 5f77f48..974ddd7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml
@@ -175,6 +175,17 @@ gpgcheck=0</value>
</property>
<property>
+ <name>stack_features</name>
+ <value></value>
+ <description>List of features supported by the stack</description>
+ <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+ <value-attributes>
+ <property-file-name>stack_features.json</property-file-name>
+ <property-file-type>json</property-file-type>
+ </value-attributes>
+ </property>
+
+ <property>
<name>stack_root</name>
<value>/usr/hdp</value>
<description>Stack root folder</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/215a61df/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
new file mode 100644
index 0000000..af0561d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -0,0 +1,15 @@
+{
+ "stack_features": [
+ { "name": "snappy", "description" : "Snappy compressor/decompressor support" , "min_version" : "2.0.0.0" , "max_version" : "2.2.0.0" }
+ ,
+ { "name": "express_upgrade", "description" : "Express upgrade support", "min_version" : "2.1.0.0" }
+ ,
+ { "name": "rolling_upgrade", "description" : "Rolling upgrade support", "min_version" : "2.2.0.0" }
+ ,
+ { "name": "config_versioning", "description" : "Configurable versions support", "min_version" : "2.3.0.0" }
+ ,
+ { "name": "ranger", "description" : "Ranger Service support", "min_version" : "2.2.0.0" }
+ ,
+ { "name": "nfs", "description" : "NFS support", "min_version" : "2.3.0.0" }
+ ]
+}
\ No newline at end of file