You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/01/17 11:41:04 UTC
git commit: AMBARI-4263. Add Falcon to the stack for
monitoring/Management and add python scripts for managing/monitoring the
service. (Eugene Chekanskiy via onishuk)
Updated Branches:
refs/heads/trunk 53f05bebb -> db379b16e
AMBARI-4263. Add Falcon to the stack for monitoring/Management and add
python scripts for managing/monitoring the service. (Eugene Chekanskiy
via onishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db379b16
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db379b16
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db379b16
Branch: refs/heads/trunk
Commit: db379b16e136b429aaf98339ad25c40f71f58570
Parents: 53f05be
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Fri Jan 17 02:39:29 2014 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Fri Jan 17 02:39:29 2014 -0800
----------------------------------------------------------------------
.../services/FALCON/configuration/global.xml | 42 ++++++
.../FALCON/configuration/oozie-site.xml | 145 +++++++++++++++++++
.../HDP/2.0.8/services/FALCON/metainfo.xml | 72 +++++++++
.../services/FALCON/package/scripts/falcon.py | 66 +++++++++
.../FALCON/package/scripts/falcon_client.py | 40 +++++
.../FALCON/package/scripts/falcon_server.py | 61 ++++++++
.../services/FALCON/package/scripts/params.py | 36 +++++
.../FALCON/package/scripts/service_check.py | 38 +++++
.../FALCON/package/scripts/status_params.py | 24 +++
.../package/templates/client.properties.j2 | 24 +++
.../package/templates/runtime.properties.j2 | 33 +++++
.../package/templates/startup.properties.j2 | 70 +++++++++
.../HDFS/package/scripts/hdfs_namenode.py | 8 +-
.../services/HDFS/package/scripts/params.py | 4 +
.../HDP/2.0.8/services/OOZIE/metainfo.xml | 5 +
.../services/OOZIE/package/scripts/oozie.py | 11 +-
.../OOZIE/package/scripts/oozie_server.py | 2 +
.../services/OOZIE/package/scripts/params.py | 5 +-
18 files changed, 679 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/global.xml
new file mode 100644
index 0000000..1d56238
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/global.xml
@@ -0,0 +1,42 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>falcon_user</name>
+ <value>falcon</value>
+ <description>Falcon user.</description>
+ </property>
+ <property>
+ <name>falcon_port</name>
+ <value>15000</value>
+ <description>Falcon server port.</description>
+ </property>
+ <property>
+ <name>falcon_local_dir</name>
+ <value>/hadoop/falcon</value>
+ <description>Falcon directory to store different data.</description>
+ </property>
+ <property>
+ <name>falcon_store_uri</name>
+ <value>file:///hadoop/falcon/store</value>
+ <description>Falcon store URI.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/oozie-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/oozie-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/oozie-site.xml
new file mode 100644
index 0000000..186677b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/configuration/oozie-site.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>oozie.service.ProxyUserService.proxyuser.falcon.hosts</name>
+ <value>*</value>
+ <description>Falcon proxyuser hosts</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ProxyUserService.proxyuser.falcon.groups</name>
+ <value>*</value>
+ <description>Falcon proxyuser groups</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-job-submit-instances</name>
+ <value>
+ now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,
+ today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,
+ yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,
+ currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,
+ lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,
+ currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,
+ lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,
+ formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,
+ latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
+ future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-action-create-inst</name>
+ <value>
+ now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst,
+ today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst,
+ yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst,
+ currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst,
+ lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst,
+ currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst,
+ lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst,
+ latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
+ future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,
+ formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-action-create</name>
+ <value>
+ now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,
+ today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,
+ yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,
+ currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,
+ lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,
+ currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,
+ lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,
+ latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
+ future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,
+ formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-job-submit-data</name>
+ <value>
+ now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo,
+ today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo,
+ yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo,
+ currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo,
+ lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo,
+ currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo,
+ lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo,
+ dataIn=org.apache.oozie.extensions.OozieELExtensions#ph1_dataIn_echo,
+ instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap,
+ formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo,
+ dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-action-start</name>
+ <value>
+ now=org.apache.oozie.extensions.OozieELExtensions#ph2_now,
+ today=org.apache.oozie.extensions.OozieELExtensions#ph2_today,
+ yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday,
+ currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth,
+ lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth,
+ currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear,
+ lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear,
+ latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest,
+ future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future,
+ dataIn=org.apache.oozie.extensions.OozieELExtensions#ph3_dataIn,
+ instanceTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime,
+ dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset,
+ formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-sla-submit</name>
+ <value>
+ instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-sla-create</name>
+ <value>
+ instanceTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime,
+ user=org.apache.oozie.coord.CoordELFunctions#coord_user
+ </value>
+ <description>Falcon</description>
+ </property>
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/metainfo.xml
new file mode 100644
index 0000000..4790cac
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/metainfo.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>FALCON</name>
+ <comment>Data management and processing platform</comment>
+ <version>0.4.0.2.0.6.0</version>
+ <components>
+ <component>
+ <name>FALCON_CLIENT</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/falcon_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>FALCON_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/falcon_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <!--TODO: uncomment this after package will be available in repo-->
+ <!--<osSpecifics>-->
+ <!--<osSpecific>-->
+ <!--<osType>any</osType>-->
+ <!--<packages>-->
+ <!--<package>-->
+ <!--<type>rpm</type>-->
+ <!--<name>falcon-0.4.0.2.0.6.0-76</name>-->
+ <!--</package>-->
+ <!--</packages>-->
+ <!--</osSpecific>-->
+ <!--</osSpecifics>-->
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <configuration-dependencies>
+ <config-type>oozie-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon.py
new file mode 100644
index 0000000..d6c3d1d
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon.py
@@ -0,0 +1,66 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+def falcon(type, action = None):
+ import params
+
+ #TODO remove after package will be available in repo
+ Execute("cd /tmp; rm -f falcon-0.4.0.2.0.6.0-76.el6.noarch.rpm; "
+ "wget http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Falcon/2.0.6.0-76/rpm/falcon-0.4.0.2.0.6.0-76.el6.noarch.rpm; "
+ "rpm -Uvh --nodeps falcon-0.4.0.2.0.6.0-76.el6.noarch.rpm",
+ not_if='yum list installed | grep falcon'
+ )
+
+ if type == 'client':
+ if action == 'config':
+ File(params.falcon_conf_dir + '/client.properties',
+ content=Template('client.properties.j2'),
+ mode=0644)
+ elif type == 'server':
+ if action == 'config':
+ Directory(params.falcon_local_dir,
+ owner=params.falcon_user,
+ recursive=True
+ )
+ Directory(params.falcon_data_dir,
+ owner=params.falcon_user,
+ recursive=True
+ )
+ File(params.falcon_conf_dir + '/runtime.properties',
+ content=Template('runtime.properties.j2'),
+ mode=0644
+ )
+ File(params.falcon_conf_dir + '/startup.properties',
+ content=Template('startup.properties.j2'),
+ mode=0644
+ )
+ if action == 'start':
+ Execute(format('env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon '
+ 'FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} '
+ '{falcon_home}/bin/falcon-start -port {falcon_port}'),
+ user=params.falcon_user
+ )
+ if action == 'stop':
+ Execute(format('env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon '
+ 'FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} '
+ '{falcon_home}/bin/falcon-stop'),
+ user=params.falcon_user
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_client.py
new file mode 100644
index 0000000..ab59b6f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_client.py
@@ -0,0 +1,40 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from falcon import falcon
+
+class FalconClient(Script):
+ def install(self, env):
+ import params
+
+ self.install_packages(env)
+ env.set_params(params)
+ self.config(env)
+
+ def config(self, env):
+ import params
+
+ falcon('client', action='config')
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ FalconClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_server.py
new file mode 100644
index 0000000..9415694
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/falcon_server.py
@@ -0,0 +1,61 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from falcon import falcon
+
+class FalconServer(Script):
+ def install(self, env):
+ import params
+
+ self.install_packages(env)
+ env.set_params(params)
+ self.config(env)
+
+ def start(self, env):
+ import params
+
+ env.set_params(params)
+
+ falcon('server', action='start')
+
+ def stop(self, env):
+ import params
+
+ env.set_params(params)
+
+ falcon('server', action='stop')
+
+
+ def config(self, env):
+ import params
+
+ env.set_params(params)
+
+ falcon('server', action='config')
+
+ def status(self, env):
+ import status_params
+
+ env.set_params(status_params)
+ check_process_status(status_params.falcon_pid_dir)
+
+
+if __name__ == "__main__":
+ FalconServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/params.py
new file mode 100644
index 0000000..7f27862
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/params.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+oozie_user = config['configurations']['global']['oozie_user']
+falcon_user = config['configurations']['global']['falcon_user']
+smoke_user = config['configurations']['global']['smokeuser']
+
+java_home = config['hostLevelParams']['java_home']
+falcon_home = '/usr/lib/falcon'
+falcon_conf_dir = '/etc/falcon/conf'
+falcon_local_dir = config['configurations']['global']['falcon_local_dir']
+falcon_log_dir = '/var/log/falcon'
+falcon_data_dir = format('{falcon_local_dir}/activemq')
+store_uri = config['configurations']['global']['falcon_store_uri']
+falcon_host = config['clusterHostInfo']['falcon_server_hosts'][0]
+falcon_port = config['configurations']['global']['falcon_port']
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/service_check.py
new file mode 100644
index 0000000..19fbaf4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/service_check.py
@@ -0,0 +1,38 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+
+class FalconServiceCheck(Script):
+
+ def service_check(self, env):
+ import params
+
+ env.set_params(params)
+
+ Execute(format("env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon "
+ "FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} "
+ "{falcon_home}/bin/falcon admin -version"),
+ user=params.smoke_user,
+ logoutput=True
+ )
+
+if __name__ == "__main__":
+ FalconServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/status_params.py
new file mode 100644
index 0000000..395766c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/scripts/status_params.py
@@ -0,0 +1,24 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+falcon_pid_dir = '/var/run/falcon'
+server_pid_file = format('{falcon_pid_dir}/falcon.pid')
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/client.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/client.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/client.properties.j2
new file mode 100644
index 0000000..6ffc110
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/client.properties.j2
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#########################################################################
+########## This is used for falcon packaging only. ###################
+## Uses default port. Please change if configured for non-default port ##
+#########################################################################
+
+falcon.url=http://{{falcon_host}}:{{falcon_port}}/
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/runtime.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/runtime.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/runtime.properties.j2
new file mode 100644
index 0000000..677e5e3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/runtime.properties.j2
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+####################################################
+#### This is used for falcon packaging only. ####
+####################################################
+
+*.domain=${falcon.app.type}
+
+*.log.cleanup.frequency.minutes.retention=hours(6)
+*.log.cleanup.frequency.hours.retention=minutes(1)
+*.log.cleanup.frequency.days.retention=days(7)
+*.log.cleanup.frequency.months.retention=months(3)
+
+#### To configure falcon servers with prism ####
+#*.all.colos=<comma separated list of colos where falcon servers are installed>
+#*.falcon.<colo>.endpoint=<falcon server endpoint>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/startup.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/startup.properties.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/startup.properties.j2
new file mode 100644
index 0000000..ade21c9
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/FALCON/package/templates/startup.properties.j2
@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+####################################################
+#### This is used for falcon packaging only. ####
+####################################################
+
+*.domain=${falcon.app.type}
+
+######### Implementation classes #########
+## DONT MODIFY UNLESS SURE ABOUT CHANGE ##
+*.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine
+*.oozie.process.workflow.builder=org.apache.falcon.workflow.OozieProcessWorkflowBuilder
+*.oozie.feed.workflow.builder=org.apache.falcon.workflow.OozieFeedWorkflowBuilder
+*.journal.impl=org.apache.falcon.transaction.SharedFileSystemJournal
+*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager
+*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService
+*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager
+*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService
+
+*.application.services=org.apache.falcon.entity.store.ConfigurationStore,\
+ org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.rerun.service.RetryService,\
+ org.apache.falcon.rerun.service.LateRunService,\
+ org.apache.falcon.service.SLAMonitoringService,\
+ org.apache.falcon.service.LogCleanupService
+prism.application.services=org.apache.falcon.entity.store.ConfigurationStore
+*.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap,\
+ org.apache.falcon.service.SharedLibraryHostingService
+prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap
+*.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory
+*.shared.libs=activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms,s4fs-0.1.jar
+
+######### Implementation classes #########
+
+*.config.store.uri={{store_uri}}
+*.system.lib.location=${falcon.home}/server/webapp/falcon/WEB-INF/lib
+prism.system.lib.location=${falcon.home}/server/webapp/prism/WEB-INF/lib
+*.broker.url=tcp://localhost:61616
+*.retry.recorder.path=${falcon.log.dir}/retry
+
+*.falcon.cleanup.service.frequency=days(1)
+
+#default time-to-live for a JMS message 3 days (time in minutes)
+*.broker.ttlInMins=4320
+*.entity.topic=FALCON.ENTITY.TOPIC
+*.max.retry.failure.count=1
+
+######### Properties for configuring iMon client and metric #########
+*.internal.queue.size=1000
+*.current.colo=default
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
index 907db3c..8b29cc3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
@@ -130,8 +130,12 @@ def create_app_directories():
mode="1777"
)
- pass
-
+ if params.has_falcon_host:
+ if params.falcon_store_uri[0:4] == "hdfs":
+ hdfs_directory(name=params.store_uri,
+ owner=params.falcon_user,
+ mode="755"
+ )
def create_user_directories():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/params.py
index a2479c0..685e25f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/params.py
@@ -32,6 +32,7 @@ dfs_datanode_keytab_file = config['configurations']['hdfs-site']['dfs.datanode.
dfs_namenode_keytab_file = config['configurations']['hdfs-site']['dfs.namenode.keytab.file']
smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab']
+falcon_user = config['configurations']['global']['falcon_user']
dfs_datanode_kerberos_principal = config['configurations']['hdfs-site']['dfs.datanode.kerberos.principal']
dfs_journalnode_kerberos_principal = config['configurations']['hdfs-site']['dfs.journalnode.kerberos.principal']
@@ -62,6 +63,7 @@ nm_host = default("/clusterHostInfo/nm_host", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", [])
+falcon_host = default("/clusterHostInfo/falcon_server_hosts", [])
has_ganglia_server = not len(ganglia_server_hosts) == 0
has_namenodes = not len(namenode_host) == 0
@@ -76,6 +78,7 @@ has_hcat_server_host = not len(hcat_server_hosts) == 0
has_hive_server_host = not len(hive_server_host) == 0
has_journalnode_hosts = not len(journalnode_hosts) == 0
has_zkfc_hosts = not len(zkfc_hosts) == 0
+has_falcon_host = not len(falcon_host) == 0
is_namenode_master = hostname in namenode_host
@@ -182,3 +185,4 @@ journalnode_address = default('/configurations/hdfs-site/dfs.journalnode.http-ad
if journalnode_address:
journalnode_port = journalnode_address.split(":")[1]
+falcon_store_uri = default('configurations/global/falcon_store_uri', None)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/metainfo.xml
index 2b08f51..79e3501 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/metainfo.xml
@@ -60,6 +60,11 @@
<type>rpm</type>
<name>extjs-2.2-1</name>
</package>
+ <!--TODO: uncomment this after package will be available in repo-->
+ <!--<package>-->
+ <!--<type>rpm</type>-->
+ <!--<name>falcon-0.4.0.2.0.6.0-76</name>-->
+ <!--</package>-->
</packages>
</osSpecific>
</osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie.py
index 26a6d16..1422d1e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie.py
@@ -23,15 +23,16 @@ from resource_management import *
def oozie(is_server=False # TODO: see if see can remove this
):
import params
-
+ #TODO hack for falcon el
+ oozie_site = dict(params.config['configurations']['oozie-site'])
+ oozie_site["oozie.services.ext"] = 'org.apache.oozie.service.JMSAccessorService,' + oozie_site["oozie.services.ext"]
XmlConfig( "oozie-site.xml",
conf_dir = params.conf_dir,
- configurations = params.config['configurations']['oozie-site'],
+ configurations = oozie_site,
owner = params.oozie_user,
group = params.user_group,
mode = 0664
)
-
Directory( params.conf_dir,
owner = params.oozie_user,
group = params.user_group
@@ -104,7 +105,9 @@ def oozie_server_specific(
cmd3 = format("cd /usr/lib/oozie && chown {oozie_user}:{user_group} {oozie_tmp_dir} && mkdir -p {oozie_libext_dir} && cp {ext_js_path} {oozie_libext_dir}")
if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
cmd3 += format(" && cp {jdbc_driver_jar} {oozie_libext_dir}")
-
+ #falcon el extension
+ if params.has_falcon_host:
+ Execute(format('cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-0.4.0.2.0.6.0-76.jar {oozie_libext_dir}'))
# this is different for HDP1
cmd4 = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-setup.sh prepare-war")
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie_server.py
index 0c4c46b..6c00738 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie_server.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/oozie_server.py
@@ -19,6 +19,8 @@ class OozieServer(Script):
def start(self, env):
import params
env.set_params(params)
+ #TODO remove this when config command will be implemented
+ self.configure(env)
oozie_service(action='start')
def stop(self, env):
http://git-wip-us.apache.org/repos/asf/ambari/blob/db379b16/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/params.py
index 06f2e0e..1ad1412 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/OOZIE/package/scripts/params.py
@@ -71,6 +71,9 @@ elif jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
else:
jdbc_driver_jar = ""
-
+hostname = config["hostname"]
+falcon_host = default("/clusterHostInfo/falcon_server_hosts", [])
+has_falcon_host = not len(falcon_host) == 0
+falcon_home = '/usr/lib/falcon'