You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2014/02/25 19:06:14 UTC
git commit: AMBARI-4809. Allow Falcon to be configured with
keytab/security and custom params (Eugene Chekanskiy via dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/trunk f18a824c3 -> 0b155481f
AMBARI-4809. Allow Falcon to be configured with keytab/security and custom params (Eugene Chekanskiy via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0b155481
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0b155481
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0b155481
Branch: refs/heads/trunk
Commit: 0b155481f1ff84294dc29fc159abd464c62c6a35
Parents: f18a824
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Tue Feb 25 20:05:03 2014 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Tue Feb 25 20:05:03 2014 +0200
----------------------------------------------------------------------
.../hooks/before-INSTALL/scripts/params.py | 3 +
.../scripts/shared_initialization.py | 6 +
.../configuration/falcon-runtime.properties.xml | 47 ++++
.../configuration/falcon-startup.properties.xml | 228 +++++++++++++++++++
.../services/FALCON/configuration/global.xml | 10 +
.../HDP/2.1.1/services/FALCON/metainfo.xml | 2 +
.../services/FALCON/package/scripts/falcon.py | 19 +-
.../services/FALCON/package/scripts/params.py | 12 +-
.../FALCON/package/scripts/service_check.py | 5 +-
.../FALCON/package/scripts/status_params.py | 2 +-
.../hooks/before-INSTALL/test_before_install.py | 3 +
.../stacks/2.1.1/FALCON/test_falcon_client.py | 6 +
.../stacks/2.1.1/FALCON/test_falcon_server.py | 64 +++---
.../stacks/2.1.1/FALCON/test_service_check.py | 19 ++
.../python/stacks/2.1.1/configs/default.json | 43 +++-
.../python/stacks/2.1.1/configs/secured.json | 48 +++-
16 files changed, 477 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
index 5fdfd50..d938e74 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
@@ -39,6 +39,7 @@ gmetad_user = config['configurations']['global']["gmetad_user"]
gmond_user = config['configurations']['global']["gmond_user"]
storm_user = config['configurations']['global']['storm_user']
tez_user = 'tez'
+falcon_user = config['configurations']['global']['falcon_user']
user_group = config['configurations']['global']['user_group']
proxyuser_group = config['configurations']['global']['proxyuser_group']
@@ -61,6 +62,7 @@ namenode_host = default("/clusterHostInfo/namenode_host", [])
zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
storm_server_hosts = default("/clusterHostInfo/nimbus_hosts", [])
+falcon_host = default('/clusterHostInfo/falcon_server_hosts', [])
has_resourcemanager = not len(rm_host) == 0
has_slaves = not len(slave_hosts) == 0
@@ -72,6 +74,7 @@ has_hbase_masters = not len(hbase_master_hosts) == 0
has_zk_host = not len(zk_hosts) == 0
has_ganglia_server = not len(ganglia_server_hosts) == 0
has_storm_server = not len(storm_server_hosts) == 0
+has_falcon_server = not len(falcon_host) == 0
is_namenode_master = hostname in namenode_host
is_jtnode_master = hostname in jtnode_host
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
index c238471..f786eed 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -101,6 +101,12 @@ def setup_users():
groups=[params.user_group]
)
+ if params.has_falcon_server:
+ User(params.falcon_user,
+ gid=params.user_group,
+ groups=[params.user_group]
+ )
+
def set_uid(user, user_dirs):
"""
user_dirs - comma separated directories
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-runtime.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-runtime.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-runtime.properties.xml
new file mode 100644
index 0000000..c194d27
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-runtime.properties.xml
@@ -0,0 +1,47 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>*.domain</name>
+ <value>${falcon.app.type}</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.log.cleanup.frequency.minutes.retention</name>
+ <value>hours(6)</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.log.cleanup.frequency.hours.retention</name>
+ <value>minutes(1)</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.log.cleanup.frequency.days.retention</name>
+ <value>days(7)</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.log.cleanup.frequency.months.retention</name>
+ <value>months(3)</value>
+ <description></description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-startup.properties.xml
new file mode 100644
index 0000000..dc12528
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/falcon-startup.properties.xml
@@ -0,0 +1,228 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>*.domain</name>
+ <value>${falcon.app.type}</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.workflow.engine.impl</name>
+ <value>org.apache.falcon.workflow.engine.OozieWorkflowEngine</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.oozie.process.workflow.builder</name>
+ <value>org.apache.falcon.workflow.OozieProcessWorkflowBuilder</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.oozie.feed.workflow.builder</name>
+ <value>org.apache.falcon.workflow.OozieFeedWorkflowBuilder</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.SchedulableEntityManager.impl</name>
+ <value>org.apache.falcon.resource.SchedulableEntityManager</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.ConfigSyncService.impl</name>
+ <value>org.apache.falcon.resource.ConfigSyncService</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.ProcessInstanceManager.impl</name>
+ <value>org.apache.falcon.resource.InstanceManager</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.catalog.service.impl</name>
+ <value>org.apache.falcon.catalog.HiveCatalogService</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.application.services</name>
+ <value>org.apache.falcon.security.AuthenticationInitializationService,\
+ org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.entity.store.ConfigurationStore,\
+ org.apache.falcon.rerun.service.RetryService,\
+ org.apache.falcon.rerun.service.LateRunService,\
+ org.apache.falcon.service.SLAMonitoringService,\
+ org.apache.falcon.service.LogCleanupService
+ </value>
+ <description></description>
+ </property>
+ <property>
+ <name>prism.application.services</name>
+ <value>org.apache.falcon.entity.store.ConfigurationStore</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.configstore.listeners</name>
+ <value>org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap,\
+ org.apache.falcon.service.SharedLibraryHostingService
+ </value>
+ <description></description>
+ </property>
+ <property>
+ <name>prism.configstore.listeners</name>
+ <value>org.apache.falcon.entity.v0.EntityGraph,\
+ org.apache.falcon.entity.ColoClusterRelation,\
+ org.apache.falcon.group.FeedGroupMap
+ </value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.broker.impl.class</name>
+ <value>org.apache.activemq.ActiveMQConnectionFactory</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.shared.libs</name>
+ <value>
+ activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms
+ </value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.config.store.uri</name>
+ <value>file://${falcon.home}/store</value>
+ <description>Location to store user entity configurations</description>
+ </property>
+ <property>
+ </property>
+ <property>
+ <name>*.system.lib.location</name>
+ <value>${falcon.home}/server/webapp/falcon/WEB-INF/lib</value>
+ <description>Location of libraries that is shipped to Hadoop</description>
+ </property>
+ <property>
+ <name>prism.system.lib.location</name>
+ <value>${falcon.home}/server/webapp/prism/WEB-INF/lib</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.retry.recorder.path</name>
+ <value>${falcon.log.dir}/retry</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.cleanup.service.frequency</name>
+ <value>days(1)</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.broker.url</name>
+ <value>tcp://localhost:61616</value>
+ <description>Default Active MQ url</description>
+ </property>
+ <property>
+ <name>*.broker.ttlInMins</name>
+ <value>4320</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.entity.topic</name>
+ <value>FALCON.ENTITY.TOPIC</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.max.retry.failure.count</name>
+ <value>1</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.internal.queue.size</name>
+ <value>1000</value>
+ <description></description>
+ </property>
+ <!--properties without default values-->
+ <property>
+ <name>*.falcon.http.authentication.cookie.domain</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.blacklisted.users</name>
+ <value></value>
+ <description>Comma separated list of black listed users</description>
+ </property>
+ <!--authentication properties-->
+ <property>
+ <name>*.falcon.authentication.type</name>
+ <value>simple</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.type</name>
+ <value>simple</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.token.validity</name>
+ <value>36000</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.signature.secret</name>
+ <value>falcon</value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>Indicates if anonymous requests are allowed when using 'simple' authentication</description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.kerberos.name.rules</name>
+ <value>DEFAULT</value>
+ <description>The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's KerberosName for more details.</description>
+ </property>
+ <!--kerberos params, must be set during security enabling-->
+ <property>
+ <name>*.falcon.service.authentication.kerberos.principal</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.falcon.service.authentication.kerberos.keytab</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
+ <name>*.dfs.namenode.kerberos.principal</name>
+ <value></value>
+ <description>name node principal to talk to config store</description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.kerberos.principal</name>
+ <value></value>
+ <description>Indicates the Kerberos principal to be used for HTTP endpoint</description>
+ </property>
+ <property>
+ <name>*.falcon.http.authentication.kerberos.keytab</name>
+ <value></value>
+ <description>Location of the keytab file with the credentials for the HTTP principal</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/global.xml
index 1d56238..b341a58 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/configuration/global.xml
@@ -30,6 +30,16 @@
<description>Falcon server port.</description>
</property>
<property>
+ <name>falcon_log_dir</name>
+ <value>/var/log/falcon</value>
+ <description>Falcon log directory.</description>
+ </property>
+ <property>
+ <name>falcon_pid_dir</name>
+ <value>/var/run/falcon</value>
+ <description>Falcon pid-file directory.</description>
+ </property>
+ <property>
<name>falcon_local_dir</name>
<value>/hadoop/falcon</value>
<description>Falcon directory to store different data.</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/metainfo.xml
index 7836401..c3bc137 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/metainfo.xml
@@ -64,6 +64,8 @@
<configuration-dependencies>
<config-type>oozie-site</config-type>
<config-type>global</config-type>
+ <config-type>falcon-startup.properties</config-type>
+ <config-type>falcon-runtime.properties</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/falcon.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/falcon.py
index 1cedbdf..ead884d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/falcon.py
@@ -21,7 +21,12 @@ from resource_management import *
def falcon(type, action = None):
import params
-
+ Directory(params.falcon_pid_dir,
+ owner=params.falcon_user
+ )
+ Directory(params.falcon_log_dir,
+ owner=params.falcon_user
+ )
if type == 'client':
if action == 'config':
File(params.falcon_conf_dir + '/client.properties',
@@ -43,13 +48,13 @@ def falcon(type, action = None):
owner=params.falcon_user,
recursive=True
)
- File(params.falcon_conf_dir + '/runtime.properties',
- content=Template('runtime.properties.j2'),
- mode=0644
+ PropertiesFile(params.falcon_conf_dir + '/runtime.properties',
+ properties=params.falcon_runtime_properties,
+ mode=0644
)
- File(params.falcon_conf_dir + '/startup.properties',
- content=Template('startup.properties.j2'),
- mode=0644
+ PropertiesFile(params.falcon_conf_dir + '/startup.properties',
+ properties=params.falcon_startup_properties,
+ mode=0644
)
if action == 'start':
Execute(format('env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon '
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/params.py
index fa7532d..69d908c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/params.py
@@ -19,8 +19,7 @@ limitations under the License.
from resource_management import *
-import status_params
-from status_params import server_pid_file
+from status_params import *
config = Script.get_config()
@@ -28,15 +27,22 @@ oozie_user = config['configurations']['global']['oozie_user']
falcon_user = config['configurations']['global']['falcon_user']
smoke_user = config['configurations']['global']['smokeuser']
+user_group = config['configurations']['global']['user_group']
+proxyuser_group = config['configurations']['global']['proxyuser_group']
+
java_home = config['hostLevelParams']['java_home']
falcon_home = '/usr/lib/falcon'
falcon_conf_dir = '/etc/falcon/conf'
falcon_local_dir = config['configurations']['global']['falcon_local_dir']
-falcon_log_dir = '/var/log/falcon'
+falcon_log_dir = config['configurations']['global']['falcon_log_dir']
falcon_data_dir = format('{falcon_local_dir}/activemq')
store_uri = config['configurations']['global']['falcon_store_uri']
falcon_host = config['clusterHostInfo']['falcon_server_hosts'][0]
falcon_port = config['configurations']['global']['falcon_port']
+falcon_runtime_properties = config['configurations']['falcon-runtime.properties']
+falcon_startup_properties = config['configurations']['falcon-startup.properties']
+http_keytab = falcon_startup_properties['*.falcon.http.authentication.kerberos.keytab']
+http_principal = falcon_startup_properties['*.falcon.http.authentication.kerberos.principal']
#for create_hdfs_directory
_authentication = config['configurations']['core-site']['hadoop.security.authentication']
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/service_check.py
index df40af7..7563d9b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/service_check.py
@@ -26,7 +26,10 @@ class FalconServiceCheck(Script):
import params
env.set_params(params)
-
+ if params.security_enabled:
+ principal_replaced = params.http_principal.replace("_HOST", params.hostname)
+ Execute(format("{kinit_path_local} -kt {http_keytab} {principal_replaced}"),
+ user=params.smoke_user)
Execute(format("env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon "
"FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} "
"{falcon_home}/bin/falcon admin -version"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/status_params.py
index 395766c..7f3aaa0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/FALCON/package/scripts/status_params.py
@@ -20,5 +20,5 @@ limitations under the License.
from resource_management import *
config = Script.get_config()
-falcon_pid_dir = '/var/run/falcon'
+falcon_pid_dir = config['configurations']['global']['falcon_pid_dir']
server_pid_file = format('{falcon_pid_dir}/falcon.pid')
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py
index 2d0754c..be5c23e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-INSTALL/test_before_install.py
@@ -84,6 +84,9 @@ class TestHookBeforeInstall(RMFTestCase):
self.assertResourceCalled('User', 'storm',
gid='hadoop',
groups=['hadoop'], )
+ self.assertResourceCalled('User', 'falcon',
+ gid='hadoop',
+ groups=['hadoop'], )
self.assertResourceCalled('Package', 'unzip', )
self.assertResourceCalled('Package', 'net-snmp', )
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_client.py b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_client.py
index a725289..b979eeb 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_client.py
+++ b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_client.py
@@ -28,6 +28,12 @@ class TestFalconClient(RMFTestCase):
command="configure",
config_file="default.json"
)
+ self.assertResourceCalled('Directory', '/var/run/falcon',
+ owner = 'falcon',
+ )
+ self.assertResourceCalled('Directory', '/var/log/falcon',
+ owner = 'falcon',
+ )
self.assertResourceCalled('File', '/etc/falcon/conf/client.properties',
content=Template('client.properties.j2'),
mode=0644, )
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_server.py
index 7dc1720..df8bd92 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_falcon_server.py
@@ -29,19 +29,13 @@ class TestFalconServer(RMFTestCase):
command="start",
config_file="default.json"
)
- self.assertResourceCalled('Directory',
- '/hadoop/falcon',
- owner='falcon',
- recursive=True, )
- self.assertResourceCalled('Directory', '/hadoop/falcon/activemq',
- owner='falcon',
- recursive=True, )
- self.assertResourceCalled('File', '/etc/falcon/conf/runtime.properties',
- content=Template('runtime.properties.j2'),
- mode=0644, )
- self.assertResourceCalled('File', '/etc/falcon/conf/startup.properties',
- content=Template('startup.properties.j2'),
- mode=0644, )
+ self.assert_configure_default()
+ self.assertResourceCalled('Directory', '/var/run/falcon',
+ owner = 'falcon',
+ )
+ self.assertResourceCalled('Directory', '/var/log/falcon',
+ owner = 'falcon',
+ )
self.assertResourceCalled('Execute',
'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR=/hadoop/falcon/activemq /usr/lib/falcon/bin/falcon-start -port 15000',
user='falcon', )
@@ -53,6 +47,12 @@ class TestFalconServer(RMFTestCase):
command="stop",
config_file="default.json"
)
+ self.assertResourceCalled('Directory', '/var/run/falcon',
+ owner = 'falcon',
+ )
+ self.assertResourceCalled('Directory', '/var/log/falcon',
+ owner = 'falcon',
+ )
self.assertResourceCalled('Execute',
'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR=/hadoop/falcon/activemq /usr/lib/falcon/bin/falcon-stop',
user='falcon', )
@@ -67,19 +67,31 @@ class TestFalconServer(RMFTestCase):
command="configure",
config_file="default.json"
)
- self.assertResourceCalled('Directory',
- '/hadoop/falcon',
- owner='falcon',
- recursive=True, )
- self.assertResourceCalled('Directory', '/hadoop/falcon/activemq',
- owner='falcon',
- recursive=True, )
- self.assertResourceCalled('File', '/etc/falcon/conf/runtime.properties',
- content=Template('runtime.properties.j2'),
- mode=0644, )
- self.assertResourceCalled('File', '/etc/falcon/conf/startup.properties',
- content=Template('startup.properties.j2'),
- mode=0644, )
+ self.assert_configure_default()
self.assertNoMoreResources()
+ def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/run/falcon',
+ owner = 'falcon',
+ )
+ self.assertResourceCalled('Directory', '/var/log/falcon',
+ owner = 'falcon',
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/hadoop/falcon/activemq',
+ owner = 'falcon',
+ recursive = True,
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/falcon/conf/runtime.properties',
+ mode = 0644,
+ properties = self.getConfig()['configurations']['falcon-runtime.properties'],
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/falcon/conf/startup.properties',
+ mode = 0644,
+ properties = self.getConfig()['configurations']['falcon-startup.properties'],
+ )
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_service_check.py b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_service_check.py
index 9ae6f67..6e9c177 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.1.1/FALCON/test_service_check.py
@@ -39,4 +39,23 @@ class TestFalconServer(RMFTestCase):
user = 'ambari-qa',
try_sleep = 20,)
self.assertNoMoreResources()
+ def test_service_check_secured(self):
+ self.executeScript("2.1.1/services/FALCON/package/scripts/service_check.py",
+ classname="FalconServiceCheck",
+ command="service_check",
+ config_file="secured.json"
+ )
+ self.assertResourceCalled('Execute','/usr/bin/kinit -kt /etc/security/keytabs/spnego.service.keytab HTTP/c6401.ambari.apache.org@EXAMPLE.COM',
+ user='ambari-qa'
+ )
+ self.assertResourceCalled('Execute', 'env '
+ 'JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon '
+ 'FALCON_PID_DIR=/var/run/falcon '
+ 'FALCON_DATA_DIR=/hadoop/falcon/activemq '
+ '/usr/lib/falcon/bin/falcon admin -version',
+ logoutput = True,
+ tries = 3,
+ user = 'ambari-qa',
+ try_sleep = 20,)
+ self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
index 89c03ab..1530293 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
@@ -30,7 +30,46 @@
"taskId": 152,
"public_hostname": "c6401.ambari.apache.org",
"configurations": {
- "mapred-site": {
+ "falcon-startup.properties": {
+ "*.oozie.feed.workflow.builder": "org.apache.falcon.workflow.OozieFeedWorkflowBuilder",
+ "*.falcon.http.authentication.token.validity": "36000",
+ "*.internal.queue.size": "1000",
+ "*.ProcessInstanceManager.impl": "org.apache.falcon.resource.InstanceManager",
+ "*.falcon.http.authentication.simple.anonymous.allowed": "true",
+ "*.broker.url": "tcp://localhost:61616",
+ "*.system.lib.location": "${falcon.home}/server/webapp/falcon/WEB-INF/lib",
+ "*.entity.topic": "FALCON.ENTITY.TOPIC",
+ "*.domain": "${falcon.app.type}",
+ "*.ConfigSyncService.impl": "org.apache.falcon.resource.ConfigSyncService",
+ "*.falcon.http.authentication.signature.secret": "falcon",
+ "*.retry.recorder.path": "${falcon.log.dir}/retry",
+ "*.broker.ttlInMins": "4320",
+ "*.falcon.authentication.type": "simple",
+ "*.catalog.service.impl": "org.apache.falcon.catalog.HiveCatalogService",
+ "prism.system.lib.location": "${falcon.home}/server/webapp/prism/WEB-INF/lib",
+ "*.falcon.http.authentication.kerberos.name.rules": "DEFAULT",
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n org.apache.falcon.service.ProcessSubscriberService,\\\n org.apache.falcon.entity.store.ConfigurationStore,\\\n org.apache.falcon.rerun.service.RetryService,\\\n org.apache.falcon.rerun.service.LateRunService,\\\n org.apache.falcon.service.SLAMonitoringService,\\\n org.apache.falcon.service.LogCleanupService",
+ "*.workflow.engine.impl": "org.apache.falcon.workflow.engine.OozieWorkflowEngine",
+ "*.SchedulableEntityManager.impl": "org.apache.falcon.resource.SchedulableEntityManager",
+ "*.max.retry.failure.count": "1",
+ "*.falcon.http.authentication.type": "simple",
+ "*.oozie.process.workflow.builder": "org.apache.falcon.workflow.OozieProcessWorkflowBuilder",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms",
+ "*.falcon.cleanup.service.frequency": "days(1)",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap",
+ "*.config.store.uri": "file://${falcon.home}/store",
+ "*.broker.impl.class": "org.apache.activemq.ActiveMQConnectionFactory",
+ "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap,\\\n org.apache.falcon.service.SharedLibraryHostingService"
+ },
+ "falcon-runtime.properties": {
+ "*.log.cleanup.frequency.hours.retention": "minutes(1)",
+ "*.log.cleanup.frequency.days.retention": "days(7)",
+ "*.log.cleanup.frequency.months.retention": "months(3)",
+ "*.log.cleanup.frequency.minutes.retention": "hours(6)",
+ "*.domain": "${falcon.app.type}"
+ },
+ "mapred-site": {
"mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020",
"mapreduce.cluster.administrators": " hadoop",
"mapreduce.reduce.input.buffer.percent": "0.0",
@@ -213,6 +252,8 @@
"templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz"
},
"global": {
+ "falcon_log_dir": "/var/log/falcon",
+ "falcon_pid_dir": "/var/run/falcon",
"security_enabled": "false",
"hbase_pid_dir": "/var/run/hbase",
"proxyuser_group": "users",
http://git-wip-us.apache.org/repos/asf/ambari/blob/0b155481/ambari-server/src/test/python/stacks/2.1.1/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1.1/configs/secured.json
index 2631592..e7e9eb6 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.1.1/configs/secured.json
@@ -35,7 +35,51 @@
"taskId": 186,
"public_hostname": "c6401.ambari.apache.org",
"configurations": {
- "mapred-site": {
+ "falcon-startup.properties": {
+ "*.oozie.feed.workflow.builder": "org.apache.falcon.workflow.OozieFeedWorkflowBuilder",
+ "*.falcon.http.authentication.token.validity": "36000",
+ "*.internal.queue.size": "1000",
+ "*.ProcessInstanceManager.impl": "org.apache.falcon.resource.InstanceManager",
+ "*.falcon.http.authentication.simple.anonymous.allowed": "true",
+ "*.falcon.http.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
+ "*.broker.url": "tcp://localhost:61616",
+ "*.system.lib.location": "${falcon.home}/server/webapp/falcon/WEB-INF/lib",
+ "*.entity.topic": "FALCON.ENTITY.TOPIC",
+ "*.domain": "${falcon.app.type}",
+ "*.dfs.namenode.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "*.falcon.http.authentication.kerberos.principal": "HTTP/c6401.ambari.apache.org@EXAMPLE.COM",
+ "*.ConfigSyncService.impl": "org.apache.falcon.resource.ConfigSyncService",
+ "*.falcon.service.authentication.kerberos.principal": "nn/_HOST@EXAMPLE.COM",
+ "*.falcon.http.authentication.signature.secret": "falcon",
+ "*.retry.recorder.path": "${falcon.log.dir}/retry",
+ "*.broker.ttlInMins": "4320",
+ "*.falcon.authentication.type": "kerberos",
+ "*.catalog.service.impl": "org.apache.falcon.catalog.HiveCatalogService",
+ "prism.system.lib.location": "${falcon.home}/server/webapp/prism/WEB-INF/lib",
+ "*.falcon.http.authentication.kerberos.name.rules": "DEFAULT",
+ "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n org.apache.falcon.service.ProcessSubscriberService,\\\n org.apache.falcon.entity.store.ConfigurationStore,\\\n org.apache.falcon.rerun.service.RetryService,\\\n org.apache.falcon.rerun.service.LateRunService,\\\n org.apache.falcon.service.SLAMonitoringService,\\\n org.apache.falcon.service.LogCleanupService",
+ "*.workflow.engine.impl": "org.apache.falcon.workflow.engine.OozieWorkflowEngine",
+ "*.SchedulableEntityManager.impl": "org.apache.falcon.resource.SchedulableEntityManager",
+ "*.max.retry.failure.count": "1",
+ "*.falcon.http.authentication.type": "kerberos",
+ "*.oozie.process.workflow.builder": "org.apache.falcon.workflow.OozieProcessWorkflowBuilder",
+ "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
+ "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms",
+ "*.falcon.cleanup.service.frequency": "days(1)",
+ "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap",
+ "*.config.store.uri": "file://${falcon.home}/store",
+ "*.broker.impl.class": "org.apache.activemq.ActiveMQConnectionFactory",
+ "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n org.apache.falcon.entity.ColoClusterRelation,\\\n org.apache.falcon.group.FeedGroupMap,\\\n org.apache.falcon.service.SharedLibraryHostingService",
+ "*.falcon.service.authentication.kerberos.keytab": "/etc/security/keytabs/nn.service.keytab"
+ },
+ "falcon-runtime.properties": {
+ "*.log.cleanup.frequency.hours.retention": "minutes(1)",
+ "*.log.cleanup.frequency.days.retention": "days(7)",
+ "*.log.cleanup.frequency.months.retention": "months(3)",
+ "*.log.cleanup.frequency.minutes.retention": "hours(6)",
+ "*.domain": "${falcon.app.type}"
+ },
+ "mapred-site": {
"mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020",
"mapreduce.jobhistory.webapp.spnego-keytab-file": "/etc/security/keytabs/spnego.service.keytab",
"mapreduce.reduce.input.buffer.percent": "0.0",
@@ -232,6 +276,8 @@
"templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz"
},
"global": {
+ "falcon_log_dir": "/var/log/falcon",
+ "falcon_pid_dir": "/var/run/falcon",
"jobhistory_http_keytab": "/etc/security/keytabs/spnego.service.keytab",
"syncLimit": "5",
"resourcemanager_principal_name": "rm/_HOST",