You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2014/10/08 20:40:24 UTC
git commit: AMBARI-7551 Create mahout definition for bigtop (rmeneses
via jaoki)
Repository: ambari
Updated Branches:
refs/heads/trunk eb563008d -> c764a574f
AMBARI-7551 Create mahout definition for bigtop (rmeneses via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c764a574
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c764a574
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c764a574
Branch: refs/heads/trunk
Commit: c764a574fe8f5c6807a9677228da3ef2938d34ec
Parents: eb56300
Author: Jun Aoki <ja...@apache.org>
Authored: Wed Oct 8 11:40:13 2014 -0700
Committer: Jun Aoki <ja...@apache.org>
Committed: Wed Oct 8 11:40:13 2014 -0700
----------------------------------------------------------------------
.../stacks/BIGTOP/0.8/role_command_order.json | 2 +
.../BIGTOP/0.8/services/MAHOUT/metainfo.xml | 66 ++++++++++++++
.../services/MAHOUT/package/scripts/mahout.py | 66 ++++++++++++++
.../MAHOUT/package/scripts/mahout_client.py | 36 ++++++++
.../services/MAHOUT/package/scripts/params.py | 55 ++++++++++++
.../MAHOUT/package/scripts/service_check.py | 92 ++++++++++++++++++++
.../MAHOUT/package/templates/mahout-env.sh.j2 | 34 ++++++++
7 files changed, 351 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json
index 2604b6d..69fcdac 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json
@@ -26,6 +26,7 @@
"HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"],
"HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"],
"PIG_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
+ "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
"SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
"ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
"ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"],
@@ -55,6 +56,7 @@
"YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
"RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"],
"PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
"NAMENODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP",
"HISTORYSERVER-STOP", "HBASE_MASTER-STOP"],
"DATANODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP",
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml
new file mode 100644
index 0000000..058f047
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>MAHOUT</name>
+ <displayName>Mahout</displayName>
+ <comment>The Apache Mahout project's goal is to build a scalable machine learning library</comment>
+ <version>0.9.666</version>
+ <components>
+ <component>
+ <name>MAHOUT</name>
+ <displayName>Mahout Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>0+</cardinality>
+ <commandScript>
+ <script>scripts/mahout_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>mahout</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py
new file mode 100644
index 0000000..3984c89
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py
@@ -0,0 +1,66 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+
+from resource_management import *
+
+def mahout():
+ import params
+
+ Directory( params.mahout_conf_dir,
+ owner = params.hdfs_user,
+ group = params.user_group
+ )
+
+ mahout_TemplateConfig( ['mahout-env.sh'])
+
+ # mahout_properties is always set to a default even if it's not in the payload
+ File(format("{mahout_conf_dir}/mahout.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.mahout_properties
+ )
+
+ if params.log4j_props:
+ File(format("{mahout_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{mahout_conf_dir}/log4j.properties"))):
+ File(format("{mahout_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user
+ )
+
+def mahout_TemplateConfig(name):
+ import params
+
+ if not isinstance(name, list):
+ name = [name]
+
+ for x in name:
+ TemplateConfig( format("{mahout_conf_dir}/{x}"),
+ owner = params.hdfs_user
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py
new file mode 100644
index 0000000..8457bfc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+Ambari Agent
+"""
+import sys
+from resource_management import *
+from mahout import mahout
+
+class MahoutClient(Script):
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ mahout()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ MahoutClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py
new file mode 100644
index 0000000..d6cd33f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py
@@ -0,0 +1,55 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+hadoop_log_dir = "/var/log/hadoop"
+mahout_conf_dir = "/etc/mahout/conf"
+hadoop_conf_dir = "/etc/hadoop/conf"
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+user_group = config['configurations']['cluster-env']['user_group']
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+hadoop_home = "/usr/lib/hadoop/"
+
+# mahout.properties - if not in the JSON command, then we need to esnure some
+# basic properties are set; this is a safety mechanism
+if (('mahout-properties' in config['configurations']) and ('mahout-content' in config['configurations']['mahout-properties'])):
+ mahout_properties = config['configurations']['mahout-properties']['mahout-content']
+else:
+ mahout_properties = "mahout.location.check.strict=false"
+
+# log4j.properties
+if (('mahout-log4j' in config['configurations']) and ('content' in config['configurations']['mahout-log4j'])):
+ log4j_props = config['configurations']['mahout-log4j']['content']
+else:
+ log4j_props = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py
new file mode 100644
index 0000000..bb2ec4d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py
@@ -0,0 +1,92 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+from resource_management import *
+
+class MahoutServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ # prepare the input file content
+ input_file = os.path.join("/tmp", "mahout_input.csv")
+ input_file_content = """\
+1,101,5.0
+1,102,5.0
+1,103,2.5
+2,101,2.0
+2,102,2.5
+2,103,5.0
+2,104,2.0
+3,101,2.5
+3,104,4.0
+3,105,4.5
+3,107,5.0"""
+
+ File(input_file, content=input_file_content, mode=644)
+
+ # create the log dir for the smoke user
+ Directory(os.path.join(params.hadoop_log_dir, params.smokeuser),
+ owner = params.smokeuser,
+ group = params.user_group,
+ mode = 755)
+
+ # transfer the input file to hdfs
+ recommenderdata_dir = "recommenderdata"
+ recommenderoutput_dir = "recommenderoutput"
+ cleanup_cmd = format("fs -rm -r {recommenderdata_dir} {recommenderoutput_dir} temp")
+ #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
+ create_file_cmd = format("{cleanup_cmd}; hadoop fs -put {input_file} {recommenderdata_dir}") #TODO: inconsistent that second command needs hadoop
+
+ test_cmd_cat = "mahout cat /etc/passwd"
+ test_cmd_recommendation = format("mahout recommenditembased --input {recommenderdata_dir} --output {recommenderoutput_dir} -s SIMILARITY_COOCCURRENCE")
+
+ ExecuteHadoop(create_file_cmd,
+ tries = 3,
+ try_sleep = 5,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ # for kinit run
+ keytab = params.smoke_user_keytab,
+ security_enabled = params.security_enabled,
+ kinit_path_local = params.kinit_path_local,
+ logoutput = True
+ )
+
+ Execute(test_cmd_cat,
+ tries = 3,
+ try_sleep = 5,
+ path = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user = params.smokeuser,
+ logoutput = True
+ )
+
+ Execute(test_cmd_recommendation,
+ tries = 3,
+ try_sleep = 5,
+ path = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user = params.smokeuser,
+ logoutput = True,
+ )
+
+if __name__ == "__main__":
+ MahoutServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/c764a574/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2
new file mode 100644
index 0000000..3f97f10
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2
@@ -0,0 +1,34 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}