You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by st...@apache.org on 2016/09/26 15:35:30 UTC
[1/4] ambari git commit: AMBARI-17891. Provide stack-advisor support
for Microsoft-R service. (Attila Doroszlai via stoader)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 57116b774 -> cd1ffc956
AMBARI-17891. Provide stack-advisor support for Microsoft-R service. (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bbb70fe1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bbb70fe1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bbb70fe1
Branch: refs/heads/branch-2.4
Commit: bbb70fe10bf5b211be8bc5b5613aa729ea79817c
Parents: 57116b7
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Wed Sep 7 16:44:21 2016 +0200
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Mon Sep 26 17:20:20 2016 +0200
----------------------------------------------------------------------
.../MICROSOFT_R/8.0.0/service_advisor.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bbb70fe1/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
index 554be2d..891d2c9 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/service_advisor.py
@@ -34,14 +34,14 @@ except Exception as e:
traceback.print_exc()
print "Failed to load parent"
-class HDP23MICROSOFT_RServiceAdvisor(service_advisor.ServiceAdvisor):
+class MICROSOFT_R800ServiceAdvisor(service_advisor.ServiceAdvisor):
- def colocateService(self, stackAdvisor, hostsComponentsMap, serviceComponents):
+ def colocateService(self, hostsComponentsMap, serviceComponents):
# colocate R_CLIENT with NODEMANAGERs and YARN_CLIENTs
rClientComponent = [component for component in serviceComponents if component["StackServiceComponents"]["component_name"] == "MICROSOFT_R_CLIENT"]
traceback.print_tb(None)
rClientComponent = rClientComponent[0]
- if not stackAdvisor.isComponentHostsPopulated(rClientComponent):
+ if not self.isComponentHostsPopulated(rClientComponent):
for hostName in hostsComponentsMap.keys():
hostComponents = hostsComponentsMap[hostName]
if ({"name": "NODEMANAGER"} in hostComponents or {"name": "YARN_CLIENT"} in hostComponents) \
@@ -51,18 +51,18 @@ class HDP23MICROSOFT_RServiceAdvisor(service_advisor.ServiceAdvisor):
and {"name": "MICROSOFT_R_CLIENT"} in hostComponents:
hostsComponentsMap[hostName].remove({"name": "MICROSOFT_R_CLIENT"})
- def getComponentLayoutValidations(self, stackAdvisor, services, hosts):
+ def getServiceComponentLayoutValidations(self, services, hosts):
componentsListList = [service["components"] for service in services["services"]]
componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
hostsCount = len(hostsList)
rClientHosts = self.getHosts(componentsList, "MICROSOFT_R_CLIENT")
- expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER") + self.getHosts(componentsList, "YARN_CLIENT"))
+ expectedrClientHosts = set(self.getHosts(componentsList, "NODEMANAGER")) | set(self.getHosts(componentsList, "YARN_CLIENT"))
items = []
- # Generate WARNING if any PXF is not colocated with NAMENODE or DATANODE
+ # Generate WARNING if any R_CLIENT is not colocated with NODEMANAGER or YARN_CLIENT
mismatchHosts = sorted(expectedrClientHosts.symmetric_difference(set(rClientHosts)))
if len(mismatchHosts) > 0:
hostsString = ', '.join(mismatchHosts)
[4/4] ambari git commit: AMBARI-18430. Microsoft-R service should be
supported on SLES11. (Balazs Bence Sari via stoader)
Posted by st...@apache.org.
AMBARI-18430. Microsoft-R service should be supported on SLES11. (Balazs Bence Sari via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cd1ffc95
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cd1ffc95
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cd1ffc95
Branch: refs/heads/branch-2.4
Commit: cd1ffc956d2739754a54e324412ac107151f60b4
Parents: 36eeb38
Author: Balazs Bence Sari <bs...@hortonworks.com>
Authored: Mon Sep 26 13:28:55 2016 +0200
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Mon Sep 26 17:34:56 2016 +0200
----------------------------------------------------------------------
.../MICROSOFT_R/8.0.0/repos/repoinfo.xml | 33 ++++++++++++++++++++
.../src/main/resources/mpack.json | 10 +-----
2 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd1ffc95/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
new file mode 100644
index 0000000..687643b
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/custom-services/MICROSOFT_R/8.0.0/repos/repoinfo.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<reposinfo>
+ <os family="redhat6">
+ <repo>
+ <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
+ <repoid>MSFT_R-8.1</repoid>
+ <reponame>MSFT_R</reponame>
+ </repo>
+ </os>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://PLEASE_ENTER_REPO_URL</baseurl>
+ <repoid>MSFT_R-8.1</repoid>
+ <reponame>MSFT_R</reponame>
+ </repo>
+ </os>
+</reposinfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd1ffc95/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
index f90ccce..52363d1 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/mpack.json
@@ -37,15 +37,7 @@
},
{
"stack_name" : "HDP",
- "stack_version" : "2.1"
- },
- {
- "stack_name" : "HDP",
- "stack_version" : "2.2"
- },
- {
- "stack_name" : "HDP",
- "stack_version" : "2.3"
+ "stack_version" : "2.0.6"
}
]
}
[2/4] ambari git commit: AMBARI-18338. Microsoft-R client should work
in a secured cluster (Balazs bence Sari via magyari_sandor)
Posted by st...@apache.org.
AMBARI-18338. Microsoft-R client should work in a secured cluster (Balazs bence Sari via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2003eff9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2003eff9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2003eff9
Branch: refs/heads/branch-2.4
Commit: 2003eff9ba6bd9ff64ee1893205476bcd6f250f7
Parents: bbb70fe
Author: Balazs Bence Sari <bs...@hortonworks.com>
Authored: Thu Sep 15 14:11:50 2016 +0200
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Mon Sep 26 17:20:50 2016 +0200
----------------------------------------------------------------------
.../MICROSOFT_R/8.0.0/kerberos.json | 15 ++++++++++
.../MICROSOFT_R/8.0.0/package/scripts/params.py | 1 +
.../8.0.0/package/scripts/params_linux.py | 29 ++++++++++++++++++++
.../8.0.0/package/scripts/service_check.py | 11 +++++++-
4 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2003eff9/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
new file mode 100644
index 0000000..86fab2d
--- /dev/null
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/kerberos.json
@@ -0,0 +1,15 @@
+{
+ "services": [
+ {
+ "name": "MICROSOFT_R",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2003eff9/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
index 36541e7..aa20797 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params.py
@@ -28,3 +28,4 @@ else:
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+revo_share_hdfs_folder = "/user/RevoShare"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/2003eff9/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
index d72c42e..71f17b6 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/params_linux.py
@@ -44,7 +44,36 @@ smoke_hdfs_user_mode = 0770
user_group = config['configurations']['cluster-env']['user_group']
security_enabled = config['configurations']['cluster-env']['security_enabled']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
# not supporting 32 bit jdk.
java64_home = config['hostLevelParams']['java_home']
+
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hdfs_user_principal = default('/configurations/hadoop-env/hdfs_principal_name', None)
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+hadoop_conf_dir = "/etc/hadoop/conf"
+
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_user_principal,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/2003eff9/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
index 8a425ca..547476b 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
@@ -27,9 +27,9 @@ from resource_management.core.resources.system import Execute, File
from resource_management.core.source import InlineTemplate, StaticFile
from resource_management.libraries.functions.format import format
from resource_management.libraries.script.script import Script
-from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
class MicrosoftRServiceCheck(Script):
pass
@@ -39,6 +39,15 @@ class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck):
import params
env.set_params(params)
+ try:
+ params.HdfsResource(params.revo_share_hdfs_folder,
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ mode=0777)
+ except Exception as exception:
+ Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception)))
+
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
Execute(kinit_cmd,
[3/4] ambari git commit: AMBARI-18349. Specify role command order to
make Microsoft-R service check run after YARN is started. (Attila Doroszlai
via stoader)
Posted by st...@apache.org.
AMBARI-18349. Specify role command order to make Microsoft-R service check run after YARN is started. (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36eeb38d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36eeb38d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36eeb38d
Branch: refs/heads/branch-2.4
Commit: 36eeb38da948abfe418ac19fe2c2286cf5041388
Parents: 2003eff
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Mon Sep 19 15:02:13 2016 +0200
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Mon Sep 26 17:21:04 2016 +0200
----------------------------------------------------------------------
.../MICROSOFT_R/8.0.0/package/scripts/service_check.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/36eeb38d/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
index 547476b..5368e94 100644
--- a/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
+++ b/contrib/management-packs/microsoft-r_mpack/src/main/resources/common-services/MICROSOFT_R/8.0.0/package/scripts/service_check.py
@@ -45,6 +45,7 @@ class MicrosoftRServiceCheckLinux(MicrosoftRServiceCheck):
action="create_on_execute",
owner=params.hdfs_user,
mode=0777)
+ params.HdfsResource(None, action="execute")
except Exception as exception:
Logger.warning("Could not check the existence of /user/RevoShare on HDFS, exception: {0}".format(str(exception)))