You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/08/05 14:17:03 UTC
ambari git commit: AMBARI-18046. Falcon Service check failed after
deleting and adding back falcon (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 80371f6ec -> 33adc47a4
AMBARI-18046. Falcon Service check failed after deleting and adding back falcon (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/33adc47a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/33adc47a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/33adc47a
Branch: refs/heads/branch-2.4
Commit: 33adc47a426b42426532c00a610bbdc52228c421
Parents: 80371f6
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Aug 5 17:16:58 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Aug 5 17:16:58 2016 +0300
----------------------------------------------------------------------
.../libraries/functions/constants.py | 2 +
.../libraries/functions/package_conditions.py | 19 ++++-
.../libraries/functions/setup_atlas_hook.py | 11 ++-
.../server/upgrade/UpgradeCatalog240.java | 18 ++++
.../0.5.0.2.1/configuration/falcon-env.xml | 2 +
.../configuration/falcon-startup.properties.xml | 2 +-
.../FALCON/0.5.0.2.1/package/scripts/falcon.py | 14 +++-
.../0.5.0.2.1/package/scripts/params_linux.py | 20 +++++
.../HDP/2.0.6/properties/stack_features.json | 11 +++
.../configuration/falcon-startup.properties.xml | 2 +-
.../configuration/falcon-startup.properties.xml | 2 +-
.../stacks/HDP/2.2/services/FALCON/metainfo.xml | 8 ++
.../stacks/HDP/2.2/upgrades/config-upgrade.xml | 2 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 36 --------
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 2 +-
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 2 +-
.../configuration/falcon-startup.properties.xml | 2 +-
.../stacks/HDP/2.5/services/FALCON/metainfo.xml | 28 +++++++
.../stacks/HDP/2.5/services/stack_advisor.py | 8 --
.../configuration/falcon-startup.properties.xml | 2 +-
.../server/upgrade/UpgradeCatalog240Test.java | 37 ++++++++-
.../stacks/2.3/common/test_stack_advisor.py | 87 --------------------
22 files changed, 172 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 0d6f2b2..c5cbb3f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -91,6 +91,8 @@ class StackFeature:
ATLAS_RANGER_PLUGIN_SUPPORT = "atlas_ranger_plugin_support"
ATLAS_UPGRADE_SUPPORT = "atlas_upgrade_support"
ATLAS_CONF_DIR_IN_PATH = "atlas_conf_dir_in_path"
+ FALCON_ATLAS_SUPPORT_2_3 = "falcon_atlas_support_2_3"
+ FALCON_ATLAS_SUPPORT = "falcon_atlas_support"
RANGER_PID_SUPPORT = "ranger_pid_support"
RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support"
RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change"
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 472b8d9..a8b5827 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -24,9 +24,12 @@ __all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_colle
import os
from resource_management.libraries.script import Script
+from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.version import format_stack_version
-def _has_applicable_local_component(config, components):
+def _has_local_components(config, components, indicator_function = any):
if 'role' not in config:
return False
if config['role'] == 'install_packages':
@@ -34,10 +37,13 @@ def _has_applicable_local_component(config, components):
# Check if
if 'localComponents' not in config:
return False
- return any([component in config['localComponents'] for component in components])
+ return indicator_function([component in config['localComponents'] for component in components])
else:
return config['role'] in components
+def _has_applicable_local_component(config, components):
+ return _has_local_components(config, components, any)
+
def should_install_lzo():
config = Script.get_config()
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
@@ -88,6 +94,15 @@ def should_install_hive_atlas():
has_atlas = len(atlas_hosts) > 0
return has_atlas
+def should_install_falcon_atlas_hook():
+ config = Script.get_config()
+ stack_version_unformatted = config['hostLevelParams']['stack_version']
+ stack_version_formatted = format_stack_version(stack_version_unformatted)
+ if check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, stack_version_formatted) \
+ or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted):
+ return _has_local_components(config, ['FALCON_SERVER', 'ATLAS_SERVER'], all)
+ return False
+
def should_install_ranger_tagsync():
config = Script.get_config()
ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
index acdb1b9..26a0dd6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
@@ -28,7 +28,9 @@ from resource_management.libraries.resources.properties_file import PropertiesFi
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.default import default
from resource_management.core.resources.system import Link
+from resource_management.core.resources.packaging import Package
from resource_management.core.logger import Logger
+from ambari_commons import OSCheck
from ambari_commons.constants import SERVICE
'''
@@ -169,4 +171,11 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
atlas_hook_file_name = os.path.join(atlas_hook_dir, file_name)
source_lib_file_name = os.path.join(jar_source_dir, file_name)
if os.path.isfile(atlas_hook_file_name):
- Link(source_lib_file_name, to=atlas_hook_file_name)
\ No newline at end of file
+ Link(source_lib_file_name, to=atlas_hook_file_name)
+
+def install_atlas_hook_packages():
+ import params
+
+ if not params.host_sys_prepped:
+ Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 8725050..5c96e5a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -2206,6 +2206,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
/**
* Updates the Falcon-related configurations for the clusters managed by this Ambari
* Removes falcon_store_uri from falcon-env.
+ * Appends '{{atlas_application_class_addition}}' to *.application.services from falcon-startup.properties if it doesn't contain it.
*
* @throws AmbariException if an error occurs while updating the configurations
*/
@@ -2215,6 +2216,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
for (final Cluster cluster : clusterMap.values()) {
+ // Remove falcon_store_uri from falcon-env.
Config falconEnvConfig = cluster.getDesiredConfigByType("falcon-env");
if (falconEnvConfig != null) {
Map<String, String> falconEnvEnvProperties = falconEnvConfig.getProperties();
@@ -2223,6 +2225,22 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
removeConfigurationPropertiesFromCluster(cluster, "falcon-env", Collections.singleton("falcon_store_uri"));
}
}
+
+ // Update falcon-startup.properties/*.application.services.
+ // Ensure {{atlas_application_class_addition}}
+ Config falconStartupConfig = cluster.getDesiredConfigByType("falcon-startup.properties");
+ if (falconStartupConfig != null) {
+ final String applicationServicesPropertyName = "*.application.services";
+ String value = falconStartupConfig.getProperties().get(applicationServicesPropertyName);
+ if (value != null) {
+ final String atlasApplicationClassAddition = "{{atlas_application_class_addition}}";
+ if (!(value.contains(atlasApplicationClassAddition) || value.contains("org.apache.falcon.atlas.service.AtlasService") || value.contains("org.apache.atlas.falcon.service.AtlasService"))) {
+ LOG.info("Appending '{}' to *.application.services from falcon-startup.properties since it doesn't contain it", atlasApplicationClassAddition);
+ String newValue = value.trim().concat(atlasApplicationClassAddition);
+ updateConfigurationPropertiesForCluster(cluster, "falcon-startup.properties", Collections.singletonMap(applicationServicesPropertyName, newValue), null, true, false);
+ }
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
index 34cf595..48984fd 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
@@ -163,7 +163,9 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}}
#export FALCON_EXPANDED_WEBAPP_DIR=
# Add the Atlas Falcon hook to the Falcon classpath
+{% if falcon_atlas_support %}
export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}
+{% endif %}
</value>
<value-attributes>
<type>content</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml
index f8681db..c0d710d 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml
@@ -70,7 +70,7 @@
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
- org.apache.falcon.service.LogCleanupService
+ org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}}
</value>
<description/>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 5712516..f214b49 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -34,8 +34,10 @@ from resource_management.libraries.script import Script
from resource_management.libraries.resources import PropertiesFile
from resource_management.libraries.functions import format
from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook
+from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, install_atlas_hook_packages, setup_atlas_jar_symlinks
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions import get_user_call_output
+from resource_management.libraries.functions import StackFeature
from ambari_commons.constants import SERVICE
from resource_management.core.logger import Logger
@@ -115,10 +117,18 @@ def falcon(type, action = None, upgrade_type=None):
cd_access = "a")
# Generate atlas-application.properties.xml file
- if has_atlas_in_cluster():
+ if params.falcon_atlas_support:
+ # If Atlas is added later than Falcon, this package will be absent.
+ install_atlas_hook_packages()
+
atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath, params.falcon_user, params.user_group)
+ # Falcon 0.10 uses FALCON_EXTRA_CLASS_PATH.
+ # Setup symlinks for older versions.
+ if params.current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, params.current_version_formatted):
+ setup_atlas_jar_symlinks("falcon", params.falcon_webinf_lib)
+
if type == 'server':
if action == 'config':
if params.store_uri[0:4] == "hdfs":
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index c302eeb..f59d77c 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -151,6 +151,8 @@ for host in atlas_hosts:
has_atlas_server_on_host = True
break
+falcon_atlas_support = False
+
# Path to add to environment variable
atlas_hook_cp = ""
if has_atlas_in_cluster():
@@ -158,12 +160,30 @@ if has_atlas_in_cluster():
# Only append /etc/atlas/conf to classpath if on HDP 2.4.* and atlas server is running on this host.
if has_atlas_server_on_host:
+ # stack_version doesn't contain a minor number of the stack (only first two numbers: 2.3). Get it from current_version_formatted
+ falcon_atlas_support = current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, current_version_formatted) \
+ or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted)
+
if check_stack_feature(StackFeature.ATLAS_CONF_DIR_IN_PATH, stack_version_formatted):
atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else format('{stack_root}/current/atlas-server/conf')
atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else format('{stack_root}/current/atlas-server')
atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook", "falcon", "*") + os.pathsep
elif check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, stack_version_formatted):
atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*')
+
+atlas_application_class_addition = ""
+if falcon_atlas_support:
+ # Some stack versions do not support Atlas Falcon hook. See stack_features.json
+ # Packaging was different in older versions.
+ if current_version_formatted and check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, current_version_formatted):
+ atlas_application_class_addition = ",\\\norg.apache.falcon.atlas.service.AtlasService"
+ atlas_plugin_package = "atlas-metadata*-falcon-plugin"
+ atlas_ubuntu_plugin_package = "atlas-metadata.*-falcon-plugin"
+ else:
+ atlas_application_class_addition = ",\\\norg.apache.atlas.falcon.service.AtlasService"
+ atlas_plugin_package = "atlas-metadata*-hive-plugin"
+ atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin"
+
#endregion
hdfs_site = config['configurations']['hdfs-site']
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index da577ac..ed995b2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -232,6 +232,17 @@
"min_version": "2.5.0.0"
},
{
+ "name": "falcon_atlas_support_2_3",
+ "description": "Falcon Atlas integration support for 2.3 stack",
+ "min_version": "2.3.99.0",
+ "max_version": "2.4.0.0"
+ },
+ {
+ "name": "falcon_atlas_support",
+ "description": "Falcon Atlas integration",
+ "min_version": "2.5.0.0"
+ },
+ {
"name": "hbase_home_directory",
"description": "Hbase home directory in HDFS needed for HBASE backup",
"min_version": "2.5.0.0"
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml
index 5ae2a63..6b954e7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/configuration/falcon-startup.properties.xml
@@ -70,7 +70,7 @@
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
- org.apache.falcon.service.LogCleanupService
+ org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}}
</value>
<description/>
<on-ambari-upgrade add="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index 694d960..19f8b50 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -35,7 +35,7 @@
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
org.apache.falcon.service.LogCleanupService,\
- org.apache.falcon.metadata.MetadataMappingService
+ org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}
</value>
<description>Falcon Services</description>
<on-ambari-upgrade add="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml
index 6b6bfd9..0b860cc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml
@@ -29,6 +29,10 @@
<package>
<name>falcon_${stack_version}</name>
</package>
+ <package>
+ <name>atlas-metadata_${stack_version}-falcon-plugin</name>
+ <condition>should_install_falcon_atlas_hook</condition>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -37,6 +41,10 @@
<package>
<name>falcon-${stack_version}</name>
</package>
+ <package>
+ <name>atlas-metadata-${stack_version}-falcon-plugin</name>
+ <condition>should_install_falcon_atlas_hook</condition>
+ </package>
</packages>
</osSpecific>
</osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
index 402e349..5f420f9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
@@ -234,7 +234,7 @@
<definition xsi:type="configure" id="hdp_2_2_0_0_falcon_application_services">
<type>falcon-startup.properties</type>
<set key="*.application.services"
- value="org.apache.falcon.security.AuthenticationInitializationService,\
 org.apache.falcon.workflow.WorkflowJobEndNotificationService,\
 org.apache.falcon.service.ProcessSubscriberService,\
 org.apache.falcon.entity.store.ConfigurationStore,\
 org.apache.falcon.rerun.service.RetryService,\
 org.apache.falcon.rerun.service.LateRunService,\
 org.apache.falcon.service.LogCleanupService,\
 org.apache.falcon.metadata.MetadataMappingService"/>
+ value="org.apache.falcon.security.AuthenticationInitializationService,\
 org.apache.falcon.workflow.WorkflowJobEndNotificationService,\
 org.apache.falcon.service.ProcessSubscriberService,\
 org.apache.falcon.entity.store.ConfigurationStore,\
 org.apache.falcon.rerun.service.RetryService,\
 org.apache.falcon.rerun.service.LateRunService,\
 org.apache.falcon.service.LogCleanupService,\
 org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/>
</definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 9f2caf9..9d75cee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -26,7 +26,6 @@ import socket
# Local Imports
from resource_management.core.logger import Logger
-from resource_management.libraries.functions.version import compare_versions
DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"}
@@ -81,7 +80,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
"KAFKA": self.recommendKAFKAConfigurations,
"RANGER": self.recommendRangerConfigurations,
"RANGER_KMS": self.recommendRangerKMSConfigurations,
- "FALCON": self.recommendFalconConfigurations,
"STORM": self.recommendStormConfigurations,
"SQOOP": self.recommendSqoopConfigurations
}
@@ -737,40 +735,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
putStormStartupProperty(notifier_plugin_property, notifier_plugin_value)
- def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
- # In 2.5, the classname changed. The 2.5 stack advisor method will
- # call this with attribute set. 2.3 will use the old method name
- if hasattr(self, "atlasFalconHookClassName"):
- atlas_application_class = self.atlasFalconHookClassName
- else:
- atlas_application_class = "org.apache.falcon.atlas.service.AtlasService"
- putFalconStartupProperty = self.putProperty(configurations, "falcon-startup.properties", services)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- # atlas
- application_services_property = "*.application.services"
- if "falcon-startup.properties" in services["configurations"] and application_services_property in services["configurations"]["falcon-startup.properties"]["properties"]:
- application_services_value = services["configurations"]["falcon-startup.properties"]["properties"][application_services_property]
- else:
- application_services_value = " "
-
- include_atlas = "ATLAS" in servicesList
- if include_atlas and atlas_application_class not in application_services_value:
- if application_services_value == " ":
- application_services_value = atlas_application_class
- else:
- application_services_value = application_services_value + "," + atlas_application_class
- if not include_atlas and atlas_application_class in application_services_value:
- application_classes = []
- for application_class in application_services_value.split(","):
- if application_class != atlas_application_class and application_class != " ":
- application_classes.append(application_class)
- if application_classes:
- application_services_value = ",".join(application_classes)
- else:
- application_services_value = " "
- putFalconStartupProperty(application_services_property, application_services_value)
-
def getServiceConfigurationValidators(self):
parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
childValidators = {
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index 7661205..9b4ef8c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -526,7 +526,7 @@
<changes>
<definition xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property">
<type>falcon-startup.properties</type>
- <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService"/>
+ <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/>
</definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index d230a68..29134a3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -345,7 +345,7 @@
<changes>
<definition xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property">
<type>falcon-startup.properties</type>
- <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService"/>
+ <set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/>
</definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml
index b0779bf..0508638 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/configuration/falcon-startup.properties.xml
@@ -37,7 +37,7 @@
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
org.apache.falcon.service.LogCleanupService,\
- org.apache.falcon.metadata.MetadataMappingService
+ org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}
</value>
<description>Falcon application services</description>
<on-ambari-upgrade add="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml
index 6586937..79296c3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/FALCON/metainfo.xml
@@ -21,6 +21,34 @@
<service>
<name>FALCON</name>
<version>0.10.0.2.5</version>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>falcon_${stack_version}</name>
+ </package>
+ <package>
+ <name>atlas-metadata_${stack_version}-hive-plugin</name>
+ <condition>should_install_falcon_atlas_hook</condition>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>falcon-${stack_version}</name>
+ </package>
+ <package>
+ <name>atlas-metadata-${stack_version}-hive-plugin</name>
+ <condition>should_install_falcon_atlas_hook</condition>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
<configuration-dependencies>
<config-type>application-properties</config-type>
</configuration-dependencies>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index d94cb09..68b03ad 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -79,13 +79,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"})
putOozieEnvProperty("oozie_admin_users", newAdminUsers)
-
- def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
- # Set the classname for the Falcon hook to 2.5 classname = all other 2.3
- # stack advisor changes are needed.
- self.atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
- super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
-
def createComponentLayoutRecommendations(self, services, hosts):
parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
services, hosts)
@@ -396,7 +389,6 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
"ATLAS": self.recommendAtlasConfigurations,
"RANGER_KMS": self.recommendRangerKMSConfigurations,
"STORM": self.recommendStormConfigurations,
- "FALCON": self.recommendFalconConfigurations,
"OOZIE": self.recommendOozieConfigurations,
"SPARK2": self.recommendSpark2Configurations
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index adc3691..c7d4304 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -34,7 +34,7 @@
org.apache.falcon.entity.store.ConfigurationStore,\
org.apache.falcon.rerun.service.RetryService,\
org.apache.falcon.rerun.service.LateRunService,\
- org.apache.falcon.service.LogCleanupService
+ org.apache.falcon.service.LogCleanupService{{atlas_application_class_addition}}
</value>
<description>Falcon Services</description>
<on-ambari-upgrade add="true"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index e125165..f7c33ad 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -784,6 +784,31 @@ public class UpgradeCatalog240Test {
propertiesExpectedFalconEnv.put("falcon_store_uri", "file:///hadoop/falcon/store");
propertiesExpectedFalconEnv.put("property", "value");
+ final String applicationServicesOldPropertyValue =
+ "org.apache.falcon.security.AuthenticationInitializationService,\\\n" +
+ " org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n" +
+ " org.apache.falcon.service.ProcessSubscriberService,\\\n" +
+ " org.apache.falcon.entity.store.ConfigurationStore,\\\n" +
+ " org.apache.falcon.rerun.service.RetryService,\\\n" +
+ " org.apache.falcon.rerun.service.LateRunService,\\\n" +
+ " org.apache.falcon.service.LogCleanupService,\\\n" +
+ " org.apache.falcon.metadata.MetadataMappingService";
+
+ final String applicationServicesExpectedPropertyValue =
+ "org.apache.falcon.security.AuthenticationInitializationService,\\\n" +
+ " org.apache.falcon.workflow.WorkflowJobEndNotificationService, \\\n" +
+ " org.apache.falcon.service.ProcessSubscriberService,\\\n" +
+ " org.apache.falcon.entity.store.ConfigurationStore,\\\n" +
+ " org.apache.falcon.rerun.service.RetryService,\\\n" +
+ " org.apache.falcon.rerun.service.LateRunService,\\\n" +
+ " org.apache.falcon.service.LogCleanupService,\\\n" +
+ " org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}";
+
+ final Config falconStartupConfig = easyMockSupport.createNiceMock(Config.class);
+
+ final Map<String, String> falconStartupConfigProperties= new HashMap<String, String>();
+ falconStartupConfigProperties.put("*.application.services", applicationServicesOldPropertyValue);
+ falconStartupConfigProperties.put("property", "value");
final Injector mockInjector = Guice.createInjector(new Module() {
@Override
public void configure(Binder binder) {
@@ -805,9 +830,16 @@ public class UpgradeCatalog240Test {
expect(mockClusterExpected.getDesiredConfigByType("falcon-env")).andReturn(mockFalconEnv).atLeastOnce();
expect(mockFalconEnv.getProperties()).andReturn(propertiesExpectedFalconEnv).anyTimes();
+ expect(mockClusterExpected.getDesiredConfigByType("falcon-startup.properties")).andReturn(falconStartupConfig).atLeastOnce();
+ expect(falconStartupConfig.getProperties()).andReturn(falconStartupConfigProperties).anyTimes();
+
Capture<Map<String, String>> falconCapture = newCapture();
expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"),
- capture(falconCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once();
+ capture(falconCapture), anyString(), (Map<String, Map<String, String>>) anyObject())).andReturn(null).once();
+
+ Capture<Map<String, String>> falconStartupCapture = newCapture();
+ expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-startup.properties"),
+ capture(falconStartupCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once();
easyMockSupport.replayAll();
mockInjector.getInstance(UpgradeCatalog240.class).updateFalconConfigs();
@@ -815,6 +847,9 @@ public class UpgradeCatalog240Test {
assertEquals("value", falconCapture.getValue().get("property"));
assertNull(falconCapture.getValue().get("falcon_store_uri"));
+
+ assertEquals("value", falconStartupCapture.getValue().get("property"));
+ assertEquals(applicationServicesExpectedPropertyValue, falconStartupCapture.getValue().get("*.application.services"));
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/33adc47a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 1af87e4..272269d 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1782,93 +1782,6 @@ class TestHDP23StackAdvisor(TestCase):
self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
-
- def test_recommendFalconConfigurations(self):
- self.maxDiff = None
- configurations = {
- "falcon-startup.properties": {
- "properties": {
- "*.application.services": "foo"
- }
- }
- }
- clusterData = {
- "cpu": 4,
- "mapMemory": 3000,
- "amMemory": 2000,
- "reduceMemory": 2056,
- "containers": 3,
- "ramPerContainer": 256
- }
- expected = {
- 'falcon-startup.properties': {
- 'properties': {
- '*.application.services': 'foo,org.apache.falcon.atlas.service.AtlasService',
- }
- }
- }
- services = {
- "services": [
- {
- "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
- "StackServices": {
- "service_name": "ATLAS",
- "service_version": "2.6.0.2.2",
- "stack_name": "HDP",
- "stack_version": "2.3"
- },
- "components": [
- {
- "StackServiceComponents": {
- "advertise_version": "false",
- "cardinality": "1",
- "component_category": "MASTER",
- "component_name": "ATLAS_SERVER",
- "display_name": "Atlas Server",
- "is_client": "false",
- "is_master": "true",
- "hostnames": []
- },
- "dependencies": []
- }
- ]
- },
- ],
- "configurations": {
- "falcon-startup.properties": {
- "properties": {
- "*.application.services": "foo"
- }
- }
- },
- "changed-configurations": [ ]
-
- }
- hosts = {
- "items" : [
- {
- "href" : "/api/v1/hosts/c6401.ambari.apache.org",
- "Hosts" : {
- "cpu_count" : 1,
- "host_name" : "c6401.ambari.apache.org",
- "os_arch" : "x86_64",
- "os_type" : "centos6",
- "ph_cpu_count" : 1,
- "public_host_name" : "c6401.ambari.apache.org",
- "rack_info" : "/default-rack",
- "total_mem" : 1922680
- }
- }
- ]
- }
-
- self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
- services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
- self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts)
- self.assertEquals(configurations, expected)
-
def test_recommendSqoopConfigurations(self):
self.maxDiff = None
configurations = {