You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/10/07 18:23:34 UTC
[1/7] ambari git commit: AMBARI-18549. oozie server start fails post
upgrade to Ambari 2.4.1 (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-18456 78a875cf2 -> 197a37f03
AMBARI-18549. oozie server start fails post upgrade to Ambari 2.4.1 (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/49c9660c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/49c9660c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/49c9660c
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 49c9660c39c379b5568f9cff2d0fad44bde03222
Parents: 56e00ac
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Oct 6 19:24:08 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 6 19:24:08 2016 +0300
----------------------------------------------------------------------
.../OOZIE/4.0.0.2.0/package/scripts/oozie_service.py | 5 +++++
.../src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py | 8 ++++++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/49c9660c/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 5c97727..3257653 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -170,6 +170,11 @@ def oozie_service(action = 'start', upgrade_type=None):
raise
elif action == 'stop':
+ Directory(params.oozie_tmp_dir,
+ owner=params.oozie_user,
+ create_parents = True,
+ )
+
stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 -force")
try:
http://git-wip-us.apache.org/repos/asf/ambari/blob/49c9660c/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index b0cc2e9..2a512f4 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -576,6 +576,10 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
+ self.assertResourceCalled('Directory', '/var/tmp/oozie',
+ owner = 'oozie',
+ create_parents = True,
+ )
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 -force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
@@ -706,6 +710,10 @@ class TestOozieServer(RMFTestCase):
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
+ self.assertResourceCalled('Directory', '/var/tmp/oozie',
+ owner = 'oozie',
+ create_parents = True,
+ )
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozied.sh stop 60 -force',
environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'",
[5/7] ambari git commit: AMBARI-18537. Update Mirosoft R version in mpack (Balázs Bence Sári via magyari_sandor)
Posted by jo...@apache.org.
AMBARI-18537. Update Mirosoft R version in mpack (Bal�zs Bence S�ri via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c153cfb6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c153cfb6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c153cfb6
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: c153cfb635fc51096c1d350ec6e23e5d5162bef7
Parents: b81deb0
Author: Bal�zs Bence S�ri <bs...@hortonworks.com>
Authored: Fri Oct 7 11:28:51 2016 +0200
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Fri Oct 7 11:28:51 2016 +0200
----------------------------------------------------------------------
.../MICROSOFT_R/8.0.0/kerberos.json | 15 ----
.../MICROSOFT_R/8.0.0/metainfo.xml | 53 ------------
.../package/files/microsoft_r_serviceCheck.r | 28 ------
.../8.0.0/package/scripts/microsoft_r.py | 47 -----------
.../MICROSOFT_R/8.0.0/package/scripts/params.py | 31 -------
.../8.0.0/package/scripts/params_linux.py | 79 -----------------
.../8.0.0/package/scripts/params_windows.py | 26 ------
.../8.0.0/package/scripts/service_check.py | 89 --------------------
.../MICROSOFT_R/8.0.0/role_command_order.json | 6 --
.../MICROSOFT_R/8.0.0/service_advisor.py | 73 ----------------
.../MICROSOFT_R/8.0.5/kerberos.json | 15 ++++
.../MICROSOFT_R/8.0.5/metainfo.xml | 53 ++++++++++++
.../package/files/microsoft_r_serviceCheck.r | 28 ++++++
.../8.0.5/package/scripts/microsoft_r.py | 47 +++++++++++
.../MICROSOFT_R/8.0.5/package/scripts/params.py | 31 +++++++
.../8.0.5/package/scripts/params_linux.py | 79 +++++++++++++++++
.../8.0.5/package/scripts/params_windows.py | 26 ++++++
.../8.0.5/package/scripts/service_check.py | 89 ++++++++++++++++++++
.../MICROSOFT_R/8.0.5/role_command_order.json | 6 ++
.../MICROSOFT_R/8.0.5/service_advisor.py | 73 ++++++++++++++++
.../MICROSOFT_R/8.0.0/metainfo.xml | 28 ------
.../MICROSOFT_R/8.0.0/repos/repoinfo.xml | 33 --------
.../MICROSOFT_R/8.0.5/metainfo.xml | 28 ++++++
.../MICROSOFT_R/8.0.5/repos/repoinfo.xml | 33 ++++++++
.../src/main/resources/mpack.json | 2 +-
25 files changed, 509 insertions(+), 509 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
deleted file mode 100644
index 86fab2d..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
- "services": [
- {
- "name": "MICROSOFT_R",
- "identities": [
- {
- "name": "/HDFS/NAMENODE/hdfs"
- },
- {
- "name": "/smokeuser"
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/metainfo.xml
deleted file mode 100644
index 709929e..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/metainfo.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>MICROSOFT_R</name>
- <displayName>Microsoft R</displayName>
- <comment>Microsoft R Server</comment>
- <version>8.0.0</version>
-
- <components>
- <component>
- <name>MICROSOFT_R_SERVER_CLIENT</name>
- <displayName>Microsoft R Server Client</displayName>
- <category>CLIENT</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/microsoft_r.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1200</timeout>
- </commandScript>
- </component>
- </components>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>YARN</service>
- </requiredServices>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/files/microsoft_r_serviceCheck.r
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/files/microsoft_r_serviceCheck.r b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/files/microsoft_r_serviceCheck.r
deleted file mode 100644
index e75c598..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/files/microsoft_r_serviceCheck.r
+++ /dev/null
@@ -1,28 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License */
-
-bigDataDirRoot <- "/tmp/share"
-myHadoopCluster <- RxHadoopMR(consoleOutput=TRUE)
-rxSetComputeContext(myHadoopCluster)
-source <- system.file("SampleData/AirlineDemoSmall.csv", package="RevoScaleR")
-inputDir <- file.path(bigDataDirRoot,"AirlineDemoSmall")
-rxHadoopMakeDir(inputDir)
-rxHadoopCopyFromLocal(source, inputDir)
-hdfsFS <- RxHdfsFileSystem()
-colInfo <- list(DayOfWeek = list(type = "factor",
-levels = c("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")))
-airDS <- RxTextData(file = inputDir, missingValueString = "M", colInfo = colInfo, fileSystem = hdfsFS)
-adsSummary <- rxSummary(~ArrDelay+CRSDepTime+DayOfWeek, data = airDS)
-adsSummary
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/microsoft_r.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/microsoft_r.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/microsoft_r.py
deleted file mode 100644
index 382bd0f..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/microsoft_r.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from resource_management.libraries.script import Script
-from resource_management.core.logger import Logger
-from resource_management.core.resources import Package
-
-
-rpms = ['microsoft-r-server-mro-8.0',
- 'microsoft-r-server-intel-mkl-8.0',
- 'microsoft-r-server-packages-8.0',
- 'microsoft-r-server-hadoop-8.0']
-
-class MicrosoftR(Script):
-
- def install(self, env):
- Logger.info('Installing R Server Client...')
- tmp_dir = Script.tmp_dir
- Logger.debug('Using temp dir: {0}'.format(tmp_dir))
-
- for rpm in rpms:
- Logger.info('Installing {0}'.format(rpm))
- Package(rpm)
-
- Logger.info('Installed R Server')
-
- def configure(self, env):
- Logger.info('Configure R Server. Nothing to do.')
-
-if __name__ == "__main__":
- MicrosoftR().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
deleted file mode 100644
index aa20797..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.default import default
-
-if OSCheck.is_windows_family():
- from params_windows import *
-else:
- from params_linux import *
-
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
-
-revo_share_hdfs_folder = "/user/RevoShare"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
deleted file mode 100644
index 71f17b6..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.version import format_stack_version
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
-version = default("/commandParams/version", None)
-
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-smoke_hdfs_user_mode = 0770
-user_group = config['configurations']['cluster-env']['user_group']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
-
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None)
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-hadoop_conf_dir = "/etc/hadoop/conf"
-
-
-import functools
-#create partial functions with common arguments for every HdfsResource call
-#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- hadoop_bin_dir = hadoop_bin_dir,
- hadoop_conf_dir = hadoop_conf_dir,
- principal_name = hdfs_user_principal,
- hdfs_site = hdfs_site,
- default_fs = default_fs,
- immutable_paths = get_not_managed_resources(),
- dfs_type = dfs_type
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_windows.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_windows.py
deleted file mode 100644
index 3eca213..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_windows.py
+++ /dev/null
@@ -1,26 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-from resource_management.libraries.script.script import Script
-
-# server configurations
-config = Script.get_config()
-
-stack_root = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
deleted file mode 100644
index 5368e94..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
+++ /dev/null
@@ -1,89 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import os
-
-from resource_management.core.exceptions import Fail
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import InlineTemplate, StaticFile
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.script.script import Script
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-
-class MicrosoftRServiceCheck(Script):
- pass
-
-@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- try:
- params.HdfsResource(params.revo_share_hdfs_folder,
- type="directory",
- action="create_on_execute",
- owner=params.hdfs_user,
- mode=0777)
- params.HdfsResource(None, action="execute")
- except Exception as exception:
- Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception)))
-
- if params.security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
- Execute(kinit_cmd,
- user=params.smokeuser
- )
-
- output_file = format('{tmp_dir}/microsoft_r_serviceCheck.out')
-
- File( format("{tmp_dir}/microsoft_r_serviceCheck.r"),
- content = StaticFile("microsoft_r_serviceCheck.r"),
- mode = 0755
- )
-
- Execute( format("Revo64 --no-save < {tmp_dir}/microsoft_r_serviceCheck.r | tee {output_file}"),
- tries = 1,
- try_sleep = 1,
- path = format('/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
- user = params.smokeuser,
- logoutput = True
- )
-
- # Verify correct output
- from resource_management.core import sudo
- output_content = sudo.read_file(format('{output_file}'))
- import re
- values_list = re.findall(r"\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+(\d*)", output_content)
- if 7 != len(values_list):
- Logger.info("Unable to verify output of service check run")
- raise Fail("Unable to verify output of service check run")
- dayCountDictionary = {'Monday': '97975', 'Tuesday': '77725', 'Wednesday': '78875', 'Thursday': '81304', 'Friday': '82987', 'Saturday': '86159', 'Sunday': '94975'}
- for (day, count) in values_list:
- if count != dayCountDictionary[day]:
- Logger.info("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count))
- raise Fail("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count))
-
-
-if __name__ == "__main__":
- MicrosoftRServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/role_command_order.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/role_command_order.json
deleted file mode 100755
index 6edbc2c..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/role_command_order.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "general_deps" : {
- "_comment" : "dependencies for Microsoft R",
- "MICROSOFT_R_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
deleted file mode 100644
index 58f0dbe..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-import fnmatch
-import imp
-import socket
-import sys
-import traceback
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
-PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
-
-try:
- with open(PARENT_FILE, 'rb') as fp:
- service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
-except Exception as e:
- traceback.print_exc()
- print "Failed to load parent"
-
-class MICROSOFT_R800ServiceAdvisor(service_advisor.ServiceAdvisor):
-
- def colocateService(self, hostsComponentsMap, serviceComponents):
- # colocate R_SERVER_CLIENT with NODEMANAGERs and YARN_CLIENTs
- rClientComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "MICROSOFT_R_SERVER_CLIENT"]
- traceback.print_tb(None)
- rClientComponent = rClientComponent[0]
- if not self.isComponentHostsPopulated(rClientComponent):
- for hostName in hostsComponentsMap.keys():
- hostComponents = hostsComponentsMap[hostName]
- if ({"name": "NODEMANAGER"} in hostComponents or {"name": "YARN_CLIENT"} in hostComponents) \
- and {"name": "MICROSOFT_R_SERVER_CLIENT"} not in hostComponents:
- hostsComponentsMap[hostName].append({ "name": "MICROSOFT_R_SERVER_CLIENT" })
- if ({"name": "NODEMANAGER"} not in hostComponents and {"name": "YARN_CLIENT"} not in hostComponents) \
- and {"name": "MICROSOFT_R_SERVER_CLIENT"} in hostComponents:
- hostsComponentsMap[hostName].remove({"name": "MICROSOFT_R_SERVER_CLIENT"})
-
- def getServiceComponentLayoutValidations(self, services, hosts):
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- hostsCount = len(hostsList)
-
- rClientHosts = self.getHosts(componentsList, "MICROSOFT_R_SERVER_CLIENT")
- expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER")) | set(self.getHosts(componentsList, "YARN_CLIENT"))
-
- items = []
-
- # Generate WARNING if any R_SERVER_CLIENT is not colocated with NODEMANAGER or YARN_CLIENT
- mismatchHosts = sorted(expectedrClientHosts.symmetric_difference(set(rClientHosts)))
- if len(mismatchHosts) > 0:
- hostsString = ', '.join(mismatchHosts)
- message = "Microsoft R Server Client must be installed on NodeManagers and YARN Clients. " \
- "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString)
- items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'MICROSOFT_R_SERVER_CLIENT' } )
-
- return items
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json
new file mode 100644
index 0000000..86fab2d
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/kerberos.json
@@ -0,0 +1,15 @@
+{
+ "services": [
+ {
+ "name": "MICROSOFT_R",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml
new file mode 100644
index 0000000..6c5f062
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/metainfo.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>MICROSOFT_R</name>
+ <displayName>Microsoft R</displayName>
+ <comment>Microsoft R Server</comment>
+ <version>8.0.5</version>
+
+ <components>
+ <component>
+ <name>MICROSOFT_R_SERVER_CLIENT</name>
+ <displayName>Microsoft R Server Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/microsoft_r.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r
new file mode 100644
index 0000000..e75c598
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/files/microsoft_r_serviceCheck.r
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License */
+
+bigDataDirRoot <- "/tmp/share"
+myHadoopCluster <- RxHadoopMR(consoleOutput=TRUE)
+rxSetComputeContext(myHadoopCluster)
+source <- system.file("SampleData/AirlineDemoSmall.csv", package="RevoScaleR")
+inputDir <- file.path(bigDataDirRoot,"AirlineDemoSmall")
+rxHadoopMakeDir(inputDir)
+rxHadoopCopyFromLocal(source, inputDir)
+hdfsFS <- RxHdfsFileSystem()
+colInfo <- list(DayOfWeek = list(type = "factor",
+levels = c("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")))
+airDS <- RxTextData(file = inputDir, missingValueString = "M", colInfo = colInfo, fileSystem = hdfsFS)
+adsSummary <- rxSummary(~ArrDelay+CRSDepTime+DayOfWeek, data = airDS)
+adsSummary
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py
new file mode 100644
index 0000000..382bd0f
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/microsoft_r.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.script import Script
+from resource_management.core.logger import Logger
+from resource_management.core.resources import Package
+
+
+rpms = ['microsoft-r-server-mro-8.0',
+ 'microsoft-r-server-intel-mkl-8.0',
+ 'microsoft-r-server-packages-8.0',
+ 'microsoft-r-server-hadoop-8.0']
+
+class MicrosoftR(Script):
+
+ def install(self, env):
+ Logger.info('Installing R Server Client...')
+ tmp_dir = Script.tmp_dir
+ Logger.debug('Using temp dir: {0}'.format(tmp_dir))
+
+ for rpm in rpms:
+ Logger.info('Installing {0}'.format(rpm))
+ Package(rpm)
+
+ Logger.info('Installed R Server')
+
+ def configure(self, env):
+ Logger.info('Configure R Server. Nothing to do.')
+
+if __name__ == "__main__":
+ MicrosoftR().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py
new file mode 100644
index 0000000..aa20797
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params.py
@@ -0,0 +1,31 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+
+revo_share_hdfs_folder = "/user/RevoShare"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py
new file mode 100644
index 0000000..71f17b6
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_linux.py
@@ -0,0 +1,79 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
+user_group = config['configurations']['cluster-env']['user_group']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None)
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+hadoop_conf_dir = "/etc/hadoop/conf"
+
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_user_principal,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py
new file mode 100644
index 0000000..3eca213
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/params_windows.py
@@ -0,0 +1,26 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from resource_management.libraries.script.script import Script
+
+# server configurations
+config = Script.get_config()
+
+stack_root = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py
new file mode 100644
index 0000000..5368e94
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/package/scripts/service_check.py
@@ -0,0 +1,89 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import InlineTemplate, StaticFile
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+
+class MicrosoftRServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ try:
+ params.HdfsResource(params.revo_share_hdfs_folder,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ mode=0777)
+ params.HdfsResource(None, action="execute")
+ except Exception as exception:
+ Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception)))
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+ Execute(kinit_cmd,
+ user=params.smokeuser
+ )
+
+ output_file = format('{tmp_dir}/microsoft_r_serviceCheck.out')
+
+ File( format("{tmp_dir}/microsoft_r_serviceCheck.r"),
+ content = StaticFile("microsoft_r_serviceCheck.r"),
+ mode = 0755
+ )
+
+ Execute( format("Revo64 --no-save < {tmp_dir}/microsoft_r_serviceCheck.r | tee {output_file}"),
+ tries = 1,
+ try_sleep = 1,
+ path = format('/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+ user = params.smokeuser,
+ logoutput = True
+ )
+
+ # Verify correct output
+ from resource_management.core import sudo
+ output_content = sudo.read_file(format('{output_file}'))
+ import re
+ values_list = re.findall(r"\s(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\s+(\d*)", output_content)
+ if 7 != len(values_list):
+ Logger.info("Unable to verify output of service check run")
+ raise Fail("Unable to verify output of service check run")
+ dayCountDictionary = {'Monday': '97975', 'Tuesday': '77725', 'Wednesday': '78875', 'Thursday': '81304', 'Friday': '82987', 'Saturday': '86159', 'Sunday': '94975'}
+ for (day, count) in values_list:
+ if count != dayCountDictionary[day]:
+ Logger.info("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count))
+ raise Fail("Service check produced incorrect output for {0}. Was expecting {1} but encountered {2}".format(day, dayCountDictionary[day], count))
+
+
+if __name__ == "__main__":
+ MicrosoftRServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json
new file mode 100755
index 0000000..6edbc2c
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/role_command_order.json
@@ -0,0 +1,6 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for Microsoft R",
+ "MICROSOFT_R_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py
new file mode 100644
index 0000000..d48afec
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.5/service_advisor.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+import fnmatch
+import imp
+import socket
+import sys
+import traceback
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class MICROSOFT_R805ServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def colocateService(self, hostsComponentsMap, serviceComponents):
+ # colocate R_SERVER_CLIENT with NODEMANAGERs and YARN_CLIENTs
+ rClientComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "MICROSOFT_R_SERVER_CLIENT"]
+ traceback.print_tb(None)
+ rClientComponent = rClientComponent[0]
+ if not self.isComponentHostsPopulated(rClientComponent):
+ for hostName in hostsComponentsMap.keys():
+ hostComponents = hostsComponentsMap[hostName]
+ if ({"name": "NODEMANAGER"} in hostComponents or {"name": "YARN_CLIENT"} in hostComponents) \
+ and {"name": "MICROSOFT_R_SERVER_CLIENT"} not in hostComponents:
+ hostsComponentsMap[hostName].append({ "name": "MICROSOFT_R_SERVER_CLIENT" })
+ if ({"name": "NODEMANAGER"} not in hostComponents and {"name": "YARN_CLIENT"} not in hostComponents) \
+ and {"name": "MICROSOFT_R_SERVER_CLIENT"} in hostComponents:
+ hostsComponentsMap[hostName].remove({"name": "MICROSOFT_R_SERVER_CLIENT"})
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ hostsCount = len(hostsList)
+
+ rClientHosts = self.getHosts(componentsList, "MICROSOFT_R_SERVER_CLIENT")
+ expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER")) | set(self.getHosts(componentsList, "YARN_CLIENT"))
+
+ items = []
+
+ # Generate WARNING if any R_SERVER_CLIENT is not colocated with NODEMANAGER or YARN_CLIENT
+ mismatchHosts = sorted(expectedrClientHosts.symmetric_difference(set(rClientHosts)))
+ if len(mismatchHosts) > 0:
+ hostsString = ', '.join(mismatchHosts)
+ message = "Microsoft R Server Client must be installed on NodeManagers and YARN Clients. " \
+ "The following {0} host(s) do not satisfy the colocation recommendation: {1}".format(len(mismatchHosts), hostsString)
+ items.append( { "type": 'host-component', "level": 'WARN', "message": message, "component-name": 'MICROSOFT_R_SERVER_CLIENT' } )
+
+ return items
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/metainfo.xml
deleted file mode 100644
index 0c69b61..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/metainfo.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>MICROSOFT_R</name>
- <extends>common-services/MICROSOFT_R/8.0.0</extends>
- <version>8.0.0.0</version>
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
deleted file mode 100644
index 687643b..0000000
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<reposinfo>
- <os family="redhat6">
- <repo>
- <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
- <repoid>MSFT_R-8.1</repoid>
- <reponame>MSFT_R</reponame>
- </repo>
- </os>
- <os family="suse11">
- <repo>
- <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
- <repoid>MSFT_R-8.1</repoid>
- <reponame>MSFT_R</reponame>
- </repo>
- </os>
-</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml
new file mode 100644
index 0000000..31d5225
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/metainfo.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>MICROSOFT_R</name>
+ <extends>common-services/MICROSOFT_R/8.0.5</extends>
+ <version>8.0.5.0</version>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml
new file mode 100644
index 0000000..deb58de
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.5/repos/repoinfo.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
+ <repoid>MSFT_R-8.0</repoid>
+ <reponame>MSFT_R</reponame>
+ </repo>
+ </os>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
+ <repoid>MSFT_R-8.0</repoid>
+ <reponame>MSFT_R</reponame>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c153cfb6/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
index 52363d1..eb7076c 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
@@ -25,7 +25,7 @@
"service_versions_map": [
{
"service_name" : "MICROSOFT_R",
- "service_version" : "8.0.0",
+ "service_version" : "8.0.5",
"applicable_stacks" : [
{
"stack_name" : "HDP",
[4/7] ambari git commit: AMBARI-18550. Views build failure. (yusaku)
Posted by jo...@apache.org.
AMBARI-18550. Views build failure. (yusaku)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b81deb0e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b81deb0e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b81deb0e
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: b81deb0e5dea0c4f1ed56567d03e0e04049372ba
Parents: 55c1dc5
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Thu Oct 6 15:30:43 2016 -0700
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Thu Oct 6 15:32:28 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/ui/hueambarimigration-view/package.json | 2 +-
contrib/views/wfmanager/src/main/resources/ui/package.json | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b81deb0e/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/package.json
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/package.json b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/package.json
index b9f832e..64b3270 100644
--- a/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/package.json
+++ b/contrib/views/hueambarimigration/src/main/resources/ui/hueambarimigration-view/package.json
@@ -40,7 +40,7 @@
"ember-cli-selectize": "0.5.3",
"ember-cli-sri": "^2.1.0",
"ember-cli-uglify": "^1.2.0",
- "ember-data": "^2.4.2",
+ "ember-data": "2.7.0",
"ember-export-application-global": "^1.0.5",
"ember-load-initializers": "^0.5.1",
"ember-power-select": "0.10.4",
http://git-wip-us.apache.org/repos/asf/ambari/blob/b81deb0e/contrib/views/wfmanager/src/main/resources/ui/package.json
----------------------------------------------------------------------
diff --git a/contrib/views/wfmanager/src/main/resources/ui/package.json b/contrib/views/wfmanager/src/main/resources/ui/package.json
index 92f698f..d04d57c 100644
--- a/contrib/views/wfmanager/src/main/resources/ui/package.json
+++ b/contrib/views/wfmanager/src/main/resources/ui/package.json
@@ -33,7 +33,7 @@
"ember-cli-release": "0.2.8",
"ember-cli-sri": "^2.0.0",
"ember-cli-uglify": "^1.2.0",
- "ember-data": "^2.3.0",
+ "ember-data": "2.7.0",
"ember-disable-proxy-controllers": "^1.0.1",
"ember-export-application-global": "^1.0.4",
"ember-load-initializers": "^0.5.0",
[3/7] ambari git commit: AMBARI-18547. No validation errors recieved
for empty or removed hadoop.proxyuser.* properties (dlysnichenko)
Posted by jo...@apache.org.
AMBARI-18547. No validation errors recieved for empty or removed hadoop.proxyuser.* properties (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/55c1dc5c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/55c1dc5c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/55c1dc5c
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 55c1dc5cb8015e3d114c186792dacd883b7aa820
Parents: cceb7d2
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Oct 6 20:01:13 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Oct 6 20:01:13 2016 +0300
----------------------------------------------------------------------
.../TestDataStructureUtils.py | 47 ++++
.../libraries/functions/data_structure_utils.py | 57 ++++
.../stacks/HDP/2.0.6/services/stack_advisor.py | 269 ++++++++++++-------
.../stacks/HDP/2.3/services/stack_advisor.py | 13 +-
.../stacks/2.0.6/common/test_stack_advisor.py | 215 +++++++++++++++
5 files changed, 495 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py b/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
new file mode 100644
index 0000000..a4bdc1c
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestDataStructureUtils.py
@@ -0,0 +1,47 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+
+from resource_management.libraries.functions.data_structure_utils import get_from_dict
+from resource_management.libraries.functions.data_structure_utils import KeyNotFound
+
+
+class TestDictUtils(TestCase):
+
+ def test_get_nested(self):
+ dict_ = {1: {2: {3: 'data'}}}
+ empty_dict = {}
+
+ self.assertEquals('data', get_from_dict(dict_, (1, 2, 3)))
+ self.assertEquals('data', get_from_dict(dict_, [1, 2, 3]))
+
+ self.assertEquals({3: 'data'}, get_from_dict(dict_, (1, 2)))
+
+ self.assertEquals({2: {3: 'data'}}, get_from_dict(dict_, 1))
+
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 2, 0)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, [1, 2, 0]))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 0, 3)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (1, 2, 3, 4)))
+ self.assertEquals(KeyNotFound, get_from_dict(dict_, (0, 2)))
+
+ self.assertEquals('default', get_from_dict(dict_, (0, 2, 3), default_value='default'))
+ self.assertEquals('default', get_from_dict(empty_dict, (0, 2, 3), default_value='default'))
+
+ self.assertEquals(KeyNotFound, get_from_dict(empty_dict, [1]))
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
new file mode 100644
index 0000000..59e01aa
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/data_structure_utils.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+__all__ = ['get_from_dict', 'convert_to_list']
+
+class KeyNotFound(object):
+ # Prohibit instantiation
+ def __new__(cls):
+ raise AttributeError("Instance creation is not supported.")
+
+def get_from_dict(dict_, keys, default_value=KeyNotFound):
+ """
+ Tries to get an element from the nested dictionaries (dict_) using the key sequence (keys).
+ For example, if keys == ['a', 'b', 'c'], will return dict_[a][b][c] if all keys are present in the map.
+ If one of the keys is absent, it returns 'default_value'.
+ """
+
+ keys = convert_to_list(keys)
+
+ curr_dict = dict_
+ if not curr_dict:
+ return default_value
+ for x in keys:
+ if isinstance(curr_dict, dict) and x in curr_dict:
+ curr_dict = curr_dict[x]
+ else:
+ return default_value
+
+ return curr_dict
+
+def convert_to_list(keys):
+ """
+ If 'keys' is a list or a tuple, this function returns it unchanged.
+ Otherwise returns a list that contains 'keys' as the only element.
+ """
+ if not isinstance(keys, (list, tuple)):
+ return [keys]
+ return keys
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 897a654..76c3112 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -26,6 +26,7 @@ from math import ceil, floor, log
from resource_management.core.logger import Logger
from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
+from resource_management.libraries.functions.data_structure_utils import get_from_dict
from stack_advisor import DefaultStackAdvisor
@@ -171,118 +172,154 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
return ambari_user
def recommendAmbariProxyUsersForHDFS(self, services, servicesList, putCoreSiteProperty, putCoreSitePropertyAttribute):
- if "HDFS" in servicesList:
- ambari_user = self.getAmbariUser(services)
- ambariHostName = socket.getfqdn()
- putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ambariHostName)
- putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
- old_ambari_user = self.getOldAmbariUser(services)
- if old_ambari_user is not None:
- putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
- putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
-
- def recommendHadoopProxyUsers (self, configurations, services, hosts):
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ if "HDFS" in servicesList:
+ ambari_user = self.getAmbariUser(services)
+ ambariHostName = socket.getfqdn()
+ putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(ambari_user), ambariHostName)
+ putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(ambari_user), "*")
+ old_ambari_user = self.getOldAmbariUser(services)
+ if old_ambari_user is not None:
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
+ putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
+
+ def getAmbariProxyUsersForHDFSValidationItems(self, properties, services):
+ validationItems = []
+ servicesList = self.get_services_list(services)
+
+ if "HDFS" in servicesList:
+ ambari_user = self.getAmbariUser(services)
+ props = (
+ "hadoop.proxyuser.{0}.hosts".format(ambari_user),
+ "hadoop.proxyuser.{0}.groups".format(ambari_user)
+ )
+ for prop in props:
+ validationItems.append({"config-name": prop, "item": self.validatorNotEmpty(properties, prop)})
+
+ return validationItems
+
+ def _getHadoopProxyUsersForService(self, serviceName, serviceUserComponents, services, hosts):
+ Logger.info("Calculating Hadoop Proxy User recommendations for {0} service.".format(serviceName))
+ servicesList = self.get_services_list(services)
+ resultUsers = {}
+
+ if serviceName in servicesList:
+ usersComponents = {}
+ for values in serviceUserComponents:
+
+ # Filter, if 4th argument is present in the tuple
+ filterFunction = values[3:]
+ if filterFunction and not filterFunction[0](services, hosts):
+ continue
+
+ userNameConfig, userNameProperty, hostSelectorMap = values[:3]
+ user = get_from_dict(services, ("configurations", userNameConfig, "properties", userNameProperty), None)
+ if user:
+ usersComponents[user] = (userNameConfig, userNameProperty, hostSelectorMap)
+
+ for user, (userNameConfig, userNameProperty, hostSelectorMap) in usersComponents.iteritems():
+ proxyUsers = {"config": userNameConfig, "propertyName": userNameProperty}
+ for proxyPropertyName, hostSelector in hostSelectorMap.iteritems():
+ componentHostNamesString = hostSelector if isinstance(hostSelector, basestring) else '*'
+ if isinstance(hostSelector, (list, tuple)):
+ componentHostNames = set()
+ for component in hostSelector:
+ componentHosts = self.getHostsWithComponent(serviceName, component, services, hosts)
+ if componentHosts is not None:
+ for componentHost in componentHosts:
+ componentHostName = componentHost["Hosts"]["host_name"]
+ componentHostNames.add(componentHostName)
+
+ componentHostNamesString = ",".join(sorted(componentHostNames))
+ Logger.info("Host List for [service='{0}'; user='{1}'; components='{2}']: {3}".format(serviceName, user, ','.join(hostSelector), componentHostNamesString))
+
+ if not proxyPropertyName in proxyUsers:
+ proxyUsers[proxyPropertyName] = componentHostNamesString
+
+ if not user in resultUsers:
+ resultUsers[user] = proxyUsers
+
+ return resultUsers
+
+ def getHadoopProxyUsers(self, services, hosts):
+ """
+ Gets Hadoop Proxy User recommendations based on the configuration that is provided by
+ getServiceHadoopProxyUsersConfigurationDict.
+
+ See getServiceHadoopProxyUsersConfigurationDict
+ """
+ servicesList = self.get_services_list(services)
users = {}
+ for serviceName, serviceUserComponents in self.getServiceHadoopProxyUsersConfigurationDict().iteritems():
+ users.update(self._getHadoopProxyUsersForService(serviceName, serviceUserComponents, services, hosts))
+
+ return users
+
+ def getServiceHadoopProxyUsersConfigurationDict(self):
+ """
+ Returns a map that is used by 'getHadoopProxyUsers' to determine service
+ user properties and related components and get proxyuser recommendations.
+ This method can be overridden in stackadvisors for the further stacks to
+ add additional services or change the previous logic.
+
+ Example of the map format:
+ {
+ "serviceName": [
+ ("configTypeName1", "userPropertyName1", {"propertyHosts": "*", "propertyGroups": "exact string value"})
+ ("configTypeName2", "userPropertyName2", {"propertyHosts": ["COMPONENT1", "COMPONENT2", "COMPONENT3"], "propertyGroups": "*"}),
+ ("configTypeName3", "userPropertyName3", {"propertyHosts": ["COMPONENT1", "COMPONENT2", "COMPONENT3"]}, filterFunction)
+ ],
+ "serviceName2": [
+ ...
+ }
+
+ If the third element of a tuple is map that maps proxy property to it's value.
+ The key could be either 'propertyHosts' or 'propertyGroups'. (Both are optional)
+ If the map value is a string, then this string will be used for the proxyuser
+ value (e.g. 'hadoop.proxyuser.{user}.hosts' = '*').
+ Otherwise map value should be alist or a tuple with component names.
+ All hosts with the provided components will be added
+ to the property (e.g. 'hadoop.proxyuser.{user}.hosts' = 'host1,host2,host3')
+
+ The forth element of the tuple is optional and if it's provided,
+ it should be a function that takes two arguments: services and hosts.
+ If it returns False, proxyusers for the tuple will not be added.
+ """
+ ALL_WILDCARD = "*"
+ HOSTS_PROPERTY = "propertyHosts"
+ GROUPS_PROPERTY = "propertyGroups"
+
+ return {
+ "HDFS": [("hadoop-env", "hdfs_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})],
+ "OOZIE": [("oozie-env", "oozie_user", {HOSTS_PROPERTY: ["OOZIE_SERVER"], GROUPS_PROPERTY: ALL_WILDCARD})],
+ "HIVE": [("hive-env", "hive_user", {HOSTS_PROPERTY: ["HIVE_SERVER", "HIVE_SERVER_INTERACTIVE"], GROUPS_PROPERTY: ALL_WILDCARD}),
+ ("hive-env", "webhcat_user", {HOSTS_PROPERTY: ["WEBHCAT_SERVER"], GROUPS_PROPERTY: ALL_WILDCARD})],
+ "YARN": [("yarn-env", "yarn_user", {HOSTS_PROPERTY: ["RESOURCEMANAGER"]}, lambda services, hosts: len(self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)) > 1)],
+ "FALCON": [("falcon-env", "falcon_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})],
+ "SPARK": [("livy-env", "livy_user", {HOSTS_PROPERTY: ALL_WILDCARD, GROUPS_PROPERTY: ALL_WILDCARD})]
+ }
+
+ def recommendHadoopProxyUsers(self, configurations, services, hosts):
+ servicesList = self.get_services_list(services)
+
if 'forced-configurations' not in services:
services["forced-configurations"] = []
- if "HDFS" in servicesList:
- hdfs_user = None
- if "hadoop-env" in services["configurations"] and "hdfs_user" in services["configurations"]["hadoop-env"]["properties"]:
- hdfs_user = services["configurations"]["hadoop-env"]["properties"]["hdfs_user"]
- if not hdfs_user in users and hdfs_user is not None:
- users[hdfs_user] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "hadoop-env", "propertyName" : "hdfs_user"}
-
- if "OOZIE" in servicesList:
- oozie_user = None
- if "oozie-env" in services["configurations"] and "oozie_user" in services["configurations"]["oozie-env"]["properties"]:
- oozie_user = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
- oozieServerrHosts = self.getHostsWithComponent("OOZIE", "OOZIE_SERVER", services, hosts)
- if oozieServerrHosts is not None:
- oozieServerHostsNameList = []
- for oozieServerHost in oozieServerrHosts:
- oozieServerHostsNameList.append(oozieServerHost["Hosts"]["host_name"])
- oozieServerHostsNames = ",".join(oozieServerHostsNameList)
- if not oozie_user in users and oozie_user is not None:
- users[oozie_user] = {"propertyHosts" : oozieServerHostsNames,"propertyGroups" : "*", "config" : "oozie-env", "propertyName" : "oozie_user"}
-
- hive_user = None
- if "HIVE" in servicesList:
- webhcat_user = None
- if "hive-env" in services["configurations"] and "hive_user" in services["configurations"]["hive-env"]["properties"] \
- and "webhcat_user" in services["configurations"]["hive-env"]["properties"]:
- hive_user = services["configurations"]["hive-env"]["properties"]["hive_user"]
- webhcat_user = services["configurations"]["hive-env"]["properties"]["webhcat_user"]
- hiveServerHosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER", services, hosts)
- hiveServerInteractiveHosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER_INTERACTIVE", services, hosts)
- webHcatServerHosts = self.getHostsWithComponent("HIVE", "WEBHCAT_SERVER", services, hosts)
-
- if hiveServerHosts is not None:
- hiveServerHostsNameList = []
- for hiveServerHost in hiveServerHosts:
- hiveServerHostsNameList.append(hiveServerHost["Hosts"]["host_name"])
- # Append Hive Server Interactive host as well, as it is Hive2/HiveServer2 component.
- if hiveServerInteractiveHosts:
- for hiveServerInteractiveHost in hiveServerInteractiveHosts:
- hiveServerInteractiveHostName = hiveServerInteractiveHost["Hosts"]["host_name"]
- if hiveServerInteractiveHostName not in hiveServerHostsNameList:
- hiveServerHostsNameList.append(hiveServerInteractiveHostName)
- Logger.info("Appended (if not exiting), Hive Server Interactive Host : '{0}', to Hive Server Host List : '{1}'".format(hiveServerInteractiveHostName, hiveServerHostsNameList))
-
- hiveServerHostsNames = ",".join(hiveServerHostsNameList) # includes Hive Server interactive host also.
- Logger.info("Hive Server and Hive Server Interactive (if enabled) Host List : {0}".format(hiveServerHostsNameList))
- if not hive_user in users and hive_user is not None:
- users[hive_user] = {"propertyHosts" : hiveServerHostsNames,"propertyGroups" : "*", "config" : "hive-env", "propertyName" : "hive_user"}
-
- if webHcatServerHosts is not None:
- webHcatServerHostsNameList = []
- for webHcatServerHost in webHcatServerHosts:
- webHcatServerHostsNameList.append(webHcatServerHost["Hosts"]["host_name"])
- webHcatServerHostsNames = ",".join(webHcatServerHostsNameList)
- if not webhcat_user in users and webhcat_user is not None:
- users[webhcat_user] = {"propertyHosts" : webHcatServerHostsNames,"propertyGroups" : "*", "config" : "hive-env", "propertyName" : "webhcat_user"}
-
- if "YARN" in servicesList:
- yarn_user = None
- if "yarn-env" in services["configurations"] and "yarn_user" in services["configurations"]["yarn-env"]["properties"]:
- yarn_user = services["configurations"]["yarn-env"]["properties"]["yarn_user"]
- rmHosts = self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)
-
- if len(rmHosts) > 1:
- rmHostsNameList = []
- for rmHost in rmHosts:
- rmHostsNameList.append(rmHost["Hosts"]["host_name"])
- rmHostsNames = ",".join(rmHostsNameList)
- if not yarn_user in users and yarn_user is not None:
- users[yarn_user] = {"propertyHosts" : rmHostsNames, "config" : "yarn-env", "propertyName" : "yarn_user"}
-
-
- if "FALCON" in servicesList:
- falconUser = None
- if "falcon-env" in services["configurations"] and "falcon_user" in services["configurations"]["falcon-env"]["properties"]:
- falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
- if not falconUser in users and falconUser is not None:
- users[falconUser] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "falcon-env", "propertyName" : "falcon_user"}
-
- if "SPARK" in servicesList:
- livyUser = None
- if "livy-env" in services["configurations"] and "livy_user" in services["configurations"]["livy-env"]["properties"]:
- livyUser = services["configurations"]["livy-env"]["properties"]["livy_user"]
- if not livyUser in users and livyUser is not None:
- users[livyUser] = {"propertyHosts" : "*","propertyGroups" : "*", "config" : "livy-env", "propertyName" : "livy_user"}
-
putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
+ users = self.getHadoopProxyUsers(services, hosts)
+
+ # Force dependencies for HIVE
+ if "HIVE" in servicesList:
+ hive_user = get_from_dict(services, ("configurations", "hive-env", "properties", "hive_user"), default_value=None)
+ if hive_user and get_from_dict(users, (hive_user, "propertyHosts"), default_value=None):
+ services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
+
for user_name, user_properties in users.iteritems():
- if hive_user and hive_user == user_name:
- if "propertyHosts" in user_properties:
- services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.hosts".format(hive_user)})
# Add properties "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" to core-site for all users
putCoreSiteProperty("hadoop.proxyuser.{0}.hosts".format(user_name) , user_properties["propertyHosts"])
- Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(hive_user, user_properties["propertyHosts"]))
+ Logger.info("Updated hadoop.proxyuser.{0}.hosts as : {1}".format(user_name, user_properties["propertyHosts"]))
if "propertyGroups" in user_properties:
putCoreSiteProperty("hadoop.proxyuser.{0}.groups".format(user_name) , user_properties["propertyGroups"])
@@ -300,6 +337,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
self.recommendAmbariProxyUsersForHDFS(services, servicesList, putCoreSiteProperty, putCoreSitePropertyAttribute)
+ def getHadoopProxyUsersValidationItems(self, properties, services, hosts):
+ validationItems = []
+ users = self.getHadoopProxyUsers(services, hosts)
+ for user_name, user_properties in users.iteritems():
+ props = ["hadoop.proxyuser.{0}.hosts".format(user_name)]
+ if "propertyGroups" in user_properties:
+ props.append("hadoop.proxyuser.{0}.groups".format(user_name))
+
+ for prop in props:
+ validationItems.append({"config-name": prop, "item": self.validatorNotEmpty(properties, prop)})
+
+ return validationItems
+
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
putHDFSSiteProperty = self.putProperty(configurations, "hdfs-site", services)
@@ -938,7 +988,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
def getServiceConfigurationValidators(self):
return {
"HDFS": { "hdfs-site": self.validateHDFSConfigurations,
- "hadoop-env": self.validateHDFSConfigurationsEnv},
+ "hadoop-env": self.validateHDFSConfigurationsEnv,
+ "core-site": self.validateHDFSConfigurationsCoreSite},
"MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
"YARN": {"yarn-site": self.validateYARNConfigurations,
"yarn-env": self.validateYARNEnvConfigurations},
@@ -1396,6 +1447,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
"for property {1}".format(recommendedValue, propertyName))
return None
+ def validatorNotEmpty(self, properties, propertyName):
+ if not propertyName in properties:
+ return self.getErrorItem("Value should be set for {0}".format(propertyName))
+ value = properties.get(propertyName)
+ if not value:
+ return self.getWarnItem("Empty value for {0}".format(propertyName))
+ return None
+
def validateMinMemorySetting(self, properties, defaultValue, propertyName):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
@@ -1511,6 +1570,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
{"config-name": 'namenode_opt_maxnewsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_opt_maxnewsize')}]
return self.toConfigurationValidationProblems(validationItems, "hadoop-env")
+ def validateHDFSConfigurationsCoreSite(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ validationItems.extend(self.getHadoopProxyUsersValidationItems(properties, services, hosts))
+ validationItems.extend(self.getAmbariProxyUsersForHDFSValidationItems(properties, services))
+ return self.toConfigurationValidationProblems(validationItems, "core-site")
+
def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):
if not propertyName in properties:
return self.getErrorItem("Value should be set")
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ee96cf8..517a6f3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -805,7 +805,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
return configurationValidationProblems
def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
+ parentValidationProblems = super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
hive_server2 = properties
validationItems = []
#Adding Ranger Plugin logic here
@@ -870,10 +870,13 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"item": self.getWarnItem(
"If Ranger Hive Plugin is disabled."\
" {0} needs to be set to {1}".format(prop_name,prop_val))})
- return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+
+ validationProblems = self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
+ validationProblems.extend(parentValidationProblems)
+ return validationProblems
def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ parentValidationProblems = super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
hbase_site = properties
validationItems = []
@@ -909,7 +912,9 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"If Ranger HBase Plugin is enabled."\
" {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
- return self.toConfigurationValidationProblems(validationItems, "hbase-site")
+ validationProblems = self.toConfigurationValidationProblems(validationItems, "hbase-site")
+ validationProblems.extend(parentValidationProblems)
+ return validationProblems
def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
kafka_broker = properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/55c1dc5c/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index cd69dc3..ec9ade5 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1620,6 +1620,221 @@ class TestHDP206StackAdvisor(TestCase):
res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
self.assertEquals(res, res_expected)
+ @patch("socket.getfqdn", new=lambda: 'test-mock-ambari-server-hostname1')
+ def test_recommendHadoopProxyUsers(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}},
+ 'oozie-env': {'properties': {'oozie_user': 'oozie-user'}},
+ 'hive-env': {'properties': {'hive_user': 'hive-user',
+ 'webhcat_user': 'webhcat-user'}},
+ 'falcon-env': {'properties': {'falcon_user': 'falcon-user'}},
+ 'livy-env': {'properties': {'livy_user': 'livy-user'}}
+ }
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'FALCON'}},
+ { 'StackServices': {'service_name': 'SPARK'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ },
+ { 'StackServices': {'service_name': 'OOZIE'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'OOZIE_SERVER',
+ 'hostnames': ['host2']
+ }
+ }
+ ]
+ },
+ { 'StackServices': {'service_name': 'HIVE'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'HIVE_SERVER',
+ 'hostnames': ['host1']
+ }
+ },
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'HIVE_SERVER_INTERACTIVE',
+ 'hostnames': ['host3']
+ }
+ },
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'WEBHCAT_SERVER',
+ 'hostnames': ['host4']
+ }
+ }
+ ]
+ },
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}},
+ {'Hosts' : {'host_name' : 'host3'}},
+ {'Hosts' : {'host_name' : 'host4'}}
+ ]
+ }
+
+ # 1) ok: check recommendations
+ expected = {
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.ambari-user.hosts': 'test-mock-ambari-server-hostname1',
+ 'hadoop.proxyuser.falcon-user.groups': '*',
+ 'hadoop.proxyuser.falcon-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hive-user.groups': '*',
+ 'hadoop.proxyuser.hive-user.hosts': 'host1,host3',
+ 'hadoop.proxyuser.livy-user.groups': '*',
+ 'hadoop.proxyuser.livy-user.hosts': '*',
+ 'hadoop.proxyuser.oozie-user.groups': '*',
+ 'hadoop.proxyuser.oozie-user.hosts': 'host2',
+ 'hadoop.proxyuser.webhcat-user.groups': '*',
+ 'hadoop.proxyuser.webhcat-user.hosts': 'host4',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2'
+ }
+
+ self.stackAdvisor.recommendHadoopProxyUsers(configurations, services, hosts)
+ self.assertEquals(expected, configurations['core-site']['properties'])
+
+ @patch("socket.getfqdn", new=lambda: 'test-mock-ambari-server-hostname1')
+ def test_validateHDFSConfigurationsCoreSite(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}}}
+ recommendedDefaults = {'hadoop.proxyuser.ambari-user.hosts': '*',
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
+ 'hadoop.proxyuser.yarn-user.groups': '*'}
+ properties = {'hadoop.proxyuser.ambari-user.hosts': '*',
+ 'hadoop.proxyuser.ambari-user.groups': '*',
+ 'hadoop.proxyuser.hdfs-user.hosts': '*',
+ 'hadoop.proxyuser.hdfs-user.groups': '*',
+ 'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
+ 'hadoop.proxyuser.yarn-user.groups': '*'}
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ }
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}}
+ ]
+ }
+
+ # 1) ok: HDFS and Ambari proxyusers are present
+ res_expected = []
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 2) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts is expected to be set
+ del properties['hadoop.proxyuser.yarn-user.hosts']
+ res_expected = [{'config-name': 'hadoop.proxyuser.yarn-user.hosts',
+ 'config-type': 'core-site',
+ 'level': 'ERROR',
+ 'message': 'Value should be set for hadoop.proxyuser.yarn-user.hosts',
+ 'type': 'configuration'}]
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 3) ok: test filter function: only one RESOURCE_MANAGER
+ services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
+ res_expected = []
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 4) fail: some proxyusers are empty or absent:
+ del properties['hadoop.proxyuser.ambari-user.hosts']
+ properties['hadoop.proxyuser.hdfs-user.groups'] = ''
+ res_expected = [{'config-name': 'hadoop.proxyuser.hdfs-user.groups',
+ 'config-type': 'core-site',
+ 'level': 'WARN',
+ 'message': 'Empty value for hadoop.proxyuser.hdfs-user.groups',
+ 'type': 'configuration'},
+ {'config-type': 'core-site',
+ 'message': 'Value should be set for hadoop.proxyuser.ambari-user.hosts',
+ 'type': 'configuration',
+ 'config-name': 'hadoop.proxyuser.ambari-user.hosts',
+ 'level': 'ERROR'}]
+ res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEquals(res, res_expected)
+
+ def test_getHadoopProxyUsers(self):
+ # input data stub
+ configurations = {'hadoop-env': {'properties': {'hdfs_user': 'hdfs-user'}},
+ 'yarn-env': {'properties': {'yarn_user': 'yarn-user'}}}
+ services = {
+ 'services': [
+ { 'StackServices': {'service_name': 'HDFS'}},
+ { 'StackServices': {'service_name': 'YARN'},
+ 'components': [
+ {
+ 'StackServiceComponents': {
+ 'component_name': 'RESOURCEMANAGER',
+ 'hostnames': ['host1', 'host2']
+ }
+ }
+ ]
+ }
+ ],
+ 'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
+ 'configurations': configurations
+ }
+ hosts = {
+ 'items' : [
+ {'Hosts' : {'host_name' : 'host1'}},
+ {'Hosts' : {'host_name' : 'host2'}}
+ ]
+ }
+
+ # 1) HDFS + YARN:
+ res_expected = {
+ 'hdfs-user': {'propertyName': 'hdfs_user', 'config': 'hadoop-env', 'propertyHosts': '*', 'propertyGroups': '*'},
+ 'yarn-user': {'propertyName': 'yarn_user', 'config': 'yarn-env', 'propertyHosts': 'host1,host2'}
+ }
+ res = self.stackAdvisor.getHadoopProxyUsers(services, hosts)
+ self.assertEquals(res, res_expected)
+
+ # 2) test filter function: only one RESOURCE_MANAGER
+ services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
+ res_expected = {
+ 'hdfs-user': {'propertyName': 'hdfs_user', 'config': 'hadoop-env', 'propertyHosts': '*', 'propertyGroups': '*'}
+ }
+ res = self.stackAdvisor.getHadoopProxyUsers(services, hosts)
+ self.assertEquals(res, res_expected)
+
def test_validateOneDataDirPerPartition(self):
recommendedDefaults = {
'dfs.datanode.du.reserved': '1024'
[7/7] ambari git commit: Merge branch 'trunk' into
branch-feature-AMBARI-18456
Posted by jo...@apache.org.
Merge branch 'trunk' into branch-feature-AMBARI-18456
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/197a37f0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/197a37f0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/197a37f0
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 197a37f034b2bc10f483d558cde7ab0f874c8d4c
Parents: 78a875c f6124a0
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Oct 7 14:21:34 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Oct 7 14:21:34 2016 -0400
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent | 17 ++
.../TestDataStructureUtils.py | 47 ++++
.../libraries/functions/data_structure_utils.py | 57 ++++
.../server/state/DependencyConditionInfo.java | 102 +++++++
.../ambari/server/state/DependencyInfo.java | 36 ++-
.../server/topology/BlueprintValidatorImpl.java | 13 +
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 44 +++
.../4.0.0.2.0/package/scripts/oozie_service.py | 5 +
.../stacks/HDP/2.0.6/services/stack_advisor.py | 269 ++++++++++++-------
.../stacks/HDP/2.3/services/stack_advisor.py | 13 +-
.../topology/BlueprintValidatorImplTest.java | 75 +++++-
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 8 +
.../stacks/2.0.6/common/test_stack_advisor.py | 215 +++++++++++++++
.../MICROSOFT_R/8.0.0/kerberos.json | 15 --
.../MICROSOFT_R/8.0.0/metainfo.xml | 53 ----
.../package/files/microsoft_r_serviceCheck.r | 28 --
.../8.0.0/package/scripts/microsoft_r.py | 47 ----
.../MICROSOFT_R/8.0.0/package/scripts/params.py | 31 ---
.../8.0.0/package/scripts/params_linux.py | 79 ------
.../8.0.0/package/scripts/params_windows.py | 26 --
.../8.0.0/package/scripts/service_check.py | 89 ------
.../MICROSOFT_R/8.0.0/role_command_order.json | 6 -
.../MICROSOFT_R/8.0.0/service_advisor.py | 73 -----
.../MICROSOFT_R/8.0.5/kerberos.json | 15 ++
.../MICROSOFT_R/8.0.5/metainfo.xml | 53 ++++
.../package/files/microsoft_r_serviceCheck.r | 28 ++
.../8.0.5/package/scripts/microsoft_r.py | 47 ++++
.../MICROSOFT_R/8.0.5/package/scripts/params.py | 31 +++
.../8.0.5/package/scripts/params_linux.py | 79 ++++++
.../8.0.5/package/scripts/params_windows.py | 26 ++
.../8.0.5/package/scripts/service_check.py | 89 ++++++
.../MICROSOFT_R/8.0.5/role_command_order.json | 6 +
.../MICROSOFT_R/8.0.5/service_advisor.py | 73 +++++
.../MICROSOFT_R/8.0.0/metainfo.xml | 28 --
.../MICROSOFT_R/8.0.0/repos/repoinfo.xml | 33 ---
.../MICROSOFT_R/8.0.5/metainfo.xml | 28 ++
.../MICROSOFT_R/8.0.5/repos/repoinfo.xml | 33 +++
.../src/main/resources/mpack.json | 2 +-
.../ui/hueambarimigration-view/package.json | 2 +-
.../src/main/resources/ui/package.json | 2 +-
40 files changed, 1304 insertions(+), 619 deletions(-)
----------------------------------------------------------------------
[2/7] ambari git commit: AMBARI-18526. Ambari breaks sudo and user
access if Ambari Agent misconfigured (dlysnichenko)
Posted by jo...@apache.org.
AMBARI-18526. Ambari breaks sudo and user access if Ambari Agent misconfigured (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cceb7d2d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cceb7d2d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cceb7d2d
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: cceb7d2d3eb8359e3a25a7c9eec5c92516f28097
Parents: 49c9660
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Oct 6 19:57:25 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Oct 6 19:57:25 2016 +0300
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cceb7d2d/ambari-agent/conf/unix/ambari-agent
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent
index ccb0b7a..30897dd 100755
--- a/ambari-agent/conf/unix/ambari-agent
+++ b/ambari-agent/conf/unix/ambari-agent
@@ -39,6 +39,17 @@ get_agent_property() {
echo $value
}
+valid_path() {
+ value=${1:?}
+ value=$(readlink -m ${value})
+ if [ "$value" == "/" ]; then
+ echo "path $1 not valid" 1>&2
+ exit 1
+ fi
+ echo $value
+}
+
+
export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH
export AMBARI_CONF_DIR=/etc/ambari-server/conf:$PATH
@@ -46,8 +57,14 @@ export AMBARI_CONF_DIR=/etc/ambari-server/conf:$PATH
export PYTHONPATH=/usr/lib/python2.6/site-packages:${PYTHONPATH:-}
export AMBARI_PID_DIR=`get_agent_property piddir`
+export AMBARI_PID_DIR=`valid_path "${AMBARI_PID_DIR:?}"`
+export AMBARI_PID_DIR="${AMBARI_PID_DIR:?}"
export AMBARI_AGENT_LOG_DIR=`get_agent_property logdir`
+export AMBARI_AGENT_LOG_DIR=`valid_path "${AMBARI_AGENT_LOG_DIR:?}"`
+export AMBARI_AGENT_LOG_DIR="${AMBARI_AGENT_LOG_DIR:?}"
KEYSDIR=`get_agent_property keysdir`
+KEYSDIR=`valid_path "${KEYSDIR:?}"`
+KEYSDIR="${KEYSDIR:?}"
AMBARI_AGENT=ambari-agent
PYTHON_WRAP=/usr/bin/ambari-python-wrap
[6/7] ambari git commit: AMBARI-18355: Introduce conditional
dependencies in stack defition to handle blueprint validation gracefully
(Amruta Borkar via dili)
Posted by jo...@apache.org.
AMBARI-18355: Introduce conditional dependencies in stack defition to handle blueprint validation gracefully (Amruta Borkar via dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f6124a05
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f6124a05
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f6124a05
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: f6124a056d2a8ed16bec917775b9d3554ab5d74d
Parents: c153cfb
Author: Di Li <di...@apache.org>
Authored: Fri Oct 7 13:34:54 2016 -0400
Committer: Di Li <di...@apache.org>
Committed: Fri Oct 7 13:34:54 2016 -0400
----------------------------------------------------------------------
.../server/state/DependencyConditionInfo.java | 102 +++++++++++++++++++
.../ambari/server/state/DependencyInfo.java | 36 ++++++-
.../server/topology/BlueprintValidatorImpl.java | 13 +++
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 44 ++++++++
.../topology/BlueprintValidatorImplTest.java | 75 +++++++++++++-
5 files changed, 268 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
new file mode 100644
index 0000000..84e186f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state;
+
+import java.util.Map;
+import java.util.Objects;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.adapters.XmlAdapter;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+import org.apache.commons.lang.NotImplementedException;
+
+/**
+ * Represents stack component dependency condition information.
+ */
+@XmlJavaTypeAdapter(DependencyConditionAdapter.class)
+public abstract class DependencyConditionInfo {
+ public abstract boolean isResolved(Map<String, Map<String, String>> properties);
+}
+
+class PropertyExistsDependencyCondition extends DependencyConditionInfo{
+
+ protected final String configType;
+ protected final String property;
+ public PropertyExistsDependencyCondition( String configType, String property) {
+ this.configType = Objects.requireNonNull(configType, "Config Type must not be null.");
+ this.property = Objects.requireNonNull(property, "Property Name must not be null.");
+ }
+
+ @Override
+ public boolean isResolved(Map<String, Map<String, String>> properties) {
+ return (properties.get(configType).containsKey(property));
+ }
+}
+
+class PropertyValueEqualsDependencyCondition extends PropertyExistsDependencyCondition {
+
+ protected final String propertyValue;
+ public PropertyValueEqualsDependencyCondition(String configType, String property, String propertyValue) {
+ super(configType, property);
+ this.propertyValue = Objects.requireNonNull(propertyValue, "Property value must not be null.");
+ }
+
+ @Override
+ public boolean isResolved(Map<String, Map<String, String>> properties) {
+ return (super.isResolved(properties) && propertyValue.equals(properties.get(configType).get(property)));
+ }
+}
+
+class DependencyConditionAdapter extends XmlAdapter<DependencyConditionAdapter.AdaptedDependencyCondition, DependencyConditionInfo> {
+
+ static class AdaptedDependencyCondition{
+ @XmlElement
+ private String configType;
+ @XmlElement
+ private String property;
+ @XmlElement
+ private String propertyValue;
+ @XmlElement(name="condition-type")
+ private String conditionType;
+ }
+
+ @Override
+ public AdaptedDependencyCondition marshal(DependencyConditionInfo arg0) throws Exception {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public DependencyConditionInfo unmarshal(AdaptedDependencyCondition adaptedDependencyCondition) throws Exception {
+ if (null == adaptedDependencyCondition) {
+ return null;
+ }
+ DependencyConditionInfo dependencyConditionInfo = null;
+ switch (adaptedDependencyCondition.conditionType) {
+ case "IF-PROPERTY-EXISTS":
+ dependencyConditionInfo = new PropertyExistsDependencyCondition(adaptedDependencyCondition.configType, adaptedDependencyCondition.property);
+ break;
+ case "PROPERTY-VALUE-EQUALS":
+ dependencyConditionInfo = new PropertyValueEqualsDependencyCondition(adaptedDependencyCondition.configType, adaptedDependencyCondition.property, adaptedDependencyCondition.propertyValue);
+ break;
+ default:
+ throw new IllegalArgumentException("Specified condition type is not not supported " + adaptedDependencyCondition.conditionType);
+ }
+ return dependencyConditionInfo;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
index e3db662..3c88401 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
@@ -20,6 +20,10 @@ package org.apache.ambari.server.state;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlElements;
+import java.util.ArrayList;
+import java.util.List;
/**
* Represents stack component dependency information.
@@ -54,7 +58,10 @@ public class DependencyInfo {
@XmlElement(name="auto-deploy")
private AutoDeployInfo m_autoDeploy;
-
+ /**
+ * Conditions for Component dependency to other components.
+ */
+ private List<DependencyConditionInfo> dependencyConditions = new ArrayList<DependencyConditionInfo>();
/**
* Setter for name property.
*
@@ -135,6 +142,33 @@ public class DependencyInfo {
public String getServiceName() {
return serviceName;
}
+ /**
+ * Get the dependencyConditions list
+ *
+ * @return dependencyConditions
+ */
+ @XmlElementWrapper(name="conditions")
+ @XmlElements(@XmlElement(name="condition"))
+ public List<DependencyConditionInfo> getDependencyConditions() {
+ return dependencyConditions;
+ }
+
+ /**
+ * Set dependencyConditions
+ *
+ * @param dependencyConditions
+ */
+ public void setDependencyConditions(List<DependencyConditionInfo> dependencyConditions) {
+ this.dependencyConditions = dependencyConditions;
+ }
+
+ /**
+ * Confirms if dependency have any condition or not
+ * @return true if dependencies are based on a condition
+ */
+ public boolean hasDependencyConditions(){
+ return !(dependencyConditions == null || dependencyConditions.isEmpty());
+ }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index a5f33ff..7bbe0db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
import org.apache.ambari.server.state.DependencyInfo;
import org.apache.ambari.server.utils.SecretReference;
import org.apache.ambari.server.utils.VersionUtils;
@@ -289,6 +290,18 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
AutoDeployInfo autoDeployInfo = dependency.getAutoDeploy();
boolean resolved = false;
+ //check if conditions are met, if any
+ if(dependency.hasDependencyConditions()) {
+ boolean conditionsSatisfied = true;
+ for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions()) {
+ if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties())) {
+ conditionsSatisfied = false;
+ }
+ }
+ if(!conditionsSatisfied){
+ continue;
+ }
+ }
if (dependencyScope.equals("cluster")) {
Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
componentName, new Cardinality("1+"), autoDeployInfo);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 65d166a..dfbbd55 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -32,6 +32,50 @@
<cardinality>1-2</cardinality>
<versionAdvertised>true</versionAdvertised>
<reassignAllowed>true</reassignAllowed>
+ <dependencies>
+ <dependency>
+ <name>HDFS/ZKFC</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition>
+ <condition-type>IF-PROPERTY-EXISTS</condition-type>
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ <dependency>
+ <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition>
+ <condition-type>IF-PROPERTY-EXISTS</condition-type>
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ <dependency>
+ <name>HDFS/JOURNALNODE</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ <conditions>
+ <condition>
+ <condition-type>IF-PROPERTY-EXISTS</condition-type>
+ <configType>hdfs-site</configType>
+ <property>dfs.nameservices</property>
+ </condition>
+ </conditions>
+ </dependency>
+ </dependencies>
<commandScript>
<script>scripts/namenode.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index b1de8ef..1501c53 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -34,7 +34,9 @@ import java.util.Map;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
import org.apache.ambari.server.state.DependencyInfo;
+import org.easymock.EasyMock;
import org.easymock.EasyMockRule;
import org.easymock.Mock;
import org.easymock.MockType;
@@ -66,15 +68,22 @@ public class BlueprintValidatorImplTest {
@Mock(type = MockType.NICE)
private DependencyInfo dependency1;
+ @Mock(type = MockType.NICE)
+ private DependencyInfo dependency2;
@Mock(type = MockType.NICE)
private ComponentInfo dependencyComponentInfo;
+ @Mock(type = MockType.NICE)
+ private DependencyConditionInfo dependencyConditionInfo1;
+ @Mock(type = MockType.NICE)
+ private DependencyConditionInfo dependencyConditionInfo2;
private final Collection<String> group1Components = new ArrayList<String>();
private final Collection<String> group2Components = new ArrayList<String>();
private final Collection<String> services = new ArrayList<String>();
private Collection<DependencyInfo> dependencies1 = new ArrayList<DependencyInfo>();
+ private List<DependencyConditionInfo> dependenciesConditionInfos1 = new ArrayList<DependencyConditionInfo>();
private AutoDeployInfo autoDeploy = new AutoDeployInfo();
private Map<String, Map<String, String>> configProperties = new HashMap<String, Map<String, String>>();
private Configuration configuration = new Configuration(configProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
@@ -105,13 +114,15 @@ public class BlueprintValidatorImplTest {
expect(stack.getCardinality("component1")).andReturn(new Cardinality("1"));
expect(stack.getCardinality("component2")).andReturn(new Cardinality("1+"));
expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+"));
+ dependenciesConditionInfos1.add(dependencyConditionInfo1);
+ dependenciesConditionInfos1.add(dependencyConditionInfo2);
expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes();
}
@After
public void tearDown() {
- reset(blueprint, stack, group1, group2, dependency1);
+ reset(blueprint, stack, group1, group2, dependency1, dependency2, dependencyConditionInfo1, dependencyConditionInfo2);
}
@Test
@@ -341,4 +352,66 @@ public class BlueprintValidatorImplTest {
verify(group1);
}
+ @Test(expected=InvalidTopologyException.class)
+ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIsSatisfied() throws Exception {
+ // GIVEN
+ hostGroups.clear();
+ hostGroups.put("group1", group1);
+
+ group1Components.add("component-1");
+ dependencies1.add(dependency1);
+ dependencies1.add(dependency2);
+ services.addAll(Collections.singleton("service-1"));
+
+
+ expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes();
+ expect(blueprint.getName()).andReturn("blueprint-1").anyTimes();
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put("yarn.resourcemanager.hostname", "testhost");
+ properties.put("yarn-site", typeProps);
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Cardinality cardinality = new Cardinality("1");
+
+ expect(stack.getComponents("service-1")).andReturn(Arrays.asList("component-1")).anyTimes();
+ expect(stack.getAutoDeployInfo("component-1")).andReturn(autoDeploy).anyTimes();
+ expect(stack.getDependenciesForComponent("component-1")).andReturn(dependencies1).anyTimes();
+ expect(stack.getCardinality("component-1")).andReturn(cardinality).anyTimes();
+
+ AutoDeployInfo dependencyAutoDeploy = null;
+
+ expect(dependency1.getScope()).andReturn("host").anyTimes();
+ expect(dependency1.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+ expect(dependency1.getComponentName()).andReturn("component-d").anyTimes();
+ expect(dependency1.getServiceName()).andReturn("service-d").anyTimes();
+ expect(dependency1.getName()).andReturn("dependency-1").anyTimes();
+ expect(dependency1.hasDependencyConditions()).andReturn(true).anyTimes();
+ expect(dependency1.getDependencyConditions()).andReturn(dependenciesConditionInfos1).anyTimes();
+ expect(dependency2.getScope()).andReturn("host").anyTimes();
+ expect(dependency2.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+ expect(dependency2.getComponentName()).andReturn("component-d").anyTimes();
+ expect(dependency2.getServiceName()).andReturn("service-d").anyTimes();
+ expect(dependency2.getName()).andReturn("dependency-2").anyTimes();
+ expect(dependency2.hasDependencyConditions()).andReturn(false).anyTimes();
+
+ expect(dependencyConditionInfo1.isResolved(EasyMock.anyObject(Map.class))).andReturn(true).anyTimes();
+ expect(dependencyConditionInfo2.isResolved(EasyMock.anyObject(Map.class))).andReturn(false).anyTimes();
+
+
+ expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes();
+ expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes();
+
+ replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2);
+
+ // WHEN
+ BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
+ validator.validateTopology();
+
+ // THEN
+ verify(group1);
+
+ }
}