You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/11/28 21:20:27 UTC
[08/24] ambari git commit: AMBARI-22467. YARN, MapReduce2, Hive,
and Oozie Should Conditionally Install LZO (aonishuk)
AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/096fedbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/096fedbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/096fedbd
Branch: refs/heads/branch-2.6
Commit: 096fedbd38ef241446ccee5020a62b62fc3f94bf
Parents: f35fd4e
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Nov 20 12:37:22 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Nov 20 12:37:22 2017 +0200
----------------------------------------------------------------------
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/get_lzo_packages.py | 50 -----------
.../libraries/functions/lzo_utils.py | 93 ++++++++++++++++++++
.../libraries/functions/package_conditions.py | 8 +-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 -------
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 6 +-
.../2.1.0.2.0/package/scripts/install_params.py | 10 ---
.../2.1.0.2.0/package/scripts/params_linux.py | 7 --
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 4 +
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 48 +++++-----
.../4.0.0.2.0/package/scripts/params_linux.py | 8 +-
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 3 +
.../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 --------
.../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 -------
.../stacks/2.2/configs/oozie-upgrade.json | 3 +-
15 files changed, 131 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index f144b2d..b907844 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -39,7 +39,7 @@ from resource_management.libraries.functions.version import *
from resource_management.libraries.functions.format_jvm_option import *
from resource_management.libraries.functions.constants import *
from resource_management.libraries.functions.get_stack_version import *
-from resource_management.libraries.functions.get_lzo_packages import *
+from resource_management.libraries.functions.lzo_utils import *
from resource_management.libraries.functions.setup_ranger_plugin import *
from resource_management.libraries.functions.curl_krb_request import *
from resource_management.libraries.functions.get_bare_principal import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
deleted file mode 100644
index cfbb7d8..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-__all__ = ["get_lzo_packages"]
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.script.script import Script
-
-# TODO: Make list of lzo packages stack driven
-def get_lzo_packages(stack_version_unformatted):
- lzo_packages = []
- script_instance = Script.get_instance()
- if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
- lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
- elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
- lzo_packages += ["lzo", "hadoop-lzo-native"]
- elif OSCheck.is_ubuntu_family():
- lzo_packages += ["liblzo2-2"]
-
- if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
- if OSCheck.is_ubuntu_family():
- lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
- script_instance.format_package_name("hadooplzo-${stack_version}-native")]
- else:
- lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
- script_instance.format_package_name("hadooplzo_${stack_version}-native")]
- else:
- lzo_packages += ["hadoop-lzo"]
-
- return lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
new file mode 100644
index 0000000..d6d987f
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+__all__ = ["should_install_lzo", "get_lzo_packages", "install_lzo_if_needed"]
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import StackFeature, stack_features
+from resource_management.libraries.script.script import Script
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.expect import expect
+from resource_management.core.resources.packaging import Package
+
+INSTALLING_LZO_WITHOUT_GPL = "Cannot install LZO. The GPL license must be explicitly enabled using 'ambari-server setup' on the Ambari host, then restart the server and try again."
+
+def get_lzo_packages():
+ lzo_packages = []
+ script_instance = Script.get_instance()
+ if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
+ lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
+ elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
+ lzo_packages += ["lzo", "hadoop-lzo-native"]
+ elif OSCheck.is_ubuntu_family():
+ lzo_packages += ["liblzo2-2"]
+
+
+ stack_version_unformatted = stack_features.get_stack_feature_version(Script.get_config()) # only used to check stack_feature, NOT as package version!
+ if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
+ if OSCheck.is_ubuntu_family():
+ lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
+ script_instance.format_package_name("hadooplzo-${stack_version}-native")]
+ else:
+ lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
+ script_instance.format_package_name("hadooplzo_${stack_version}-native")]
+ else:
+ lzo_packages += ["hadoop-lzo"]
+
+ return lzo_packages
+
+def should_install_lzo():
+ """
+ Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
+ """
+ config = Script.get_config()
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+
+ if not lzo_enabled:
+ return False
+
+ is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False)
+ if not is_gpl_license_accepted:
+ Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
+ return False
+
+ return True
+
+def install_lzo_if_needed():
+ """
+ Install lzo package if {#should_install_lzo} is true
+ """
+ if not should_install_lzo():
+ return
+
+ lzo_packages = get_lzo_packages()
+
+ config = Script.get_config()
+ agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+ agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
+ Package(lzo_packages,
+ retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
+ retry_count=agent_stack_retry_count
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 31e78b9..ded63cf 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -19,7 +19,7 @@ limitations under the License.
Ambari Agent
"""
-__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
+__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
"should_install_mysql", "should_install_ranger_tagsync"]
import os
@@ -44,12 +44,6 @@ def _has_local_components(config, components, indicator_function = any):
def _has_applicable_local_component(config, components):
return _has_local_components(config, components, any)
-def should_install_lzo():
- config = Script.get_config()
- io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
- lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- return lzo_enabled
-
def should_install_phoenix():
phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 2df1b9f..2ab5ef8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -203,11 +203,6 @@
<package>
<name>hadoop</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
@@ -224,16 +219,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -252,16 +237,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -280,11 +255,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index 6f702d3..23aa08c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -23,6 +23,7 @@ from resource_management import *
import os
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def hdfs(name=None):
@@ -133,10 +134,7 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
+ install_lzo_if_needed()
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
index fe488c3..dc3279f 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
@@ -23,17 +23,7 @@ if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
- from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
-
- # The logic for LZO also exists in OOZIE's params.py
- io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
- lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- lzo_packages = get_lzo_packages(stack_version_unformatted)
-
- exclude_packages = []
- if not lzo_enabled:
- exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 5ca012b..5e1f4ac 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -383,12 +382,6 @@ HdfsResource = functools.partial(
immutable_paths = get_not_managed_resources(),
dfs_type = dfs_type
)
-
-
-# The logic for LZO also exists in OOZIE's params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
name_node_params = default("/commandParams/namenode", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index abbe59e..fbff099 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -41,6 +41,7 @@ from resource_management.core.logger import Logger
from resource_management.core import utils
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook
from resource_management.libraries.functions.security_commons import update_credential_provider_path
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from ambari_commons.constants import SERVICE
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -105,6 +106,9 @@ def hive(name=None):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def hive(name=None):
import params
+
+ install_lzo_if_needed()
+
hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf")
# Permissions 644 for conf dir (client) files, and 600 for conf.server
mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 7d22603..2316f79 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f
from resource_management.libraries.functions.oozie_prepare_war import prepare_war
from resource_management.libraries.functions.copy_tarball import get_current_version
from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.security_commons import update_credential_provider_path
from resource_management.core.resources.packaging import Package
@@ -185,16 +186,21 @@ def oozie(is_server=False, upgrade_type=None):
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
)
- pass
oozie_ownership()
-
- if is_server:
+
+ if params.lzo_enabled:
+ install_lzo_if_needed()
+ Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ not_if = no_op_test,
+ )
+
+ if is_server:
oozie_server_specific(upgrade_type)
-
+
def oozie_ownership():
import params
-
+
File ( format("{conf_dir}/hadoop-config.xml"),
owner = params.oozie_user,
group = params.user_group
@@ -237,14 +243,14 @@ def get_oozie_ext_zip_source_paths(upgrade_type, params):
def oozie_server_specific(upgrade_type):
import params
-
+
no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
-
+
File(params.pid_file,
action="delete",
not_if=no_op_test
)
-
+
oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir]
Directory( oozie_server_directories,
owner = params.oozie_user,
@@ -253,25 +259,25 @@ def oozie_server_specific(upgrade_type):
create_parents = True,
cd_access="a",
)
-
+
Directory(params.oozie_libext_dir,
create_parents = True,
)
-
+
hashcode_file = format("{oozie_home}/.hashcode")
skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
Execute( untar_sharelib, # time-expensive
- not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
+ not_if = format("{no_op_test} || {skip_recreate_sharelib}"),
sudo = True,
)
configure_cmds = []
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
-
+
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
@@ -285,8 +291,8 @@ def oozie_server_specific(upgrade_type):
sudo=True,
)
break
-
-
+
+
Directory(params.oozie_webapps_conf_dir,
owner = params.oozie_user,
group = params.user_group,
@@ -305,14 +311,6 @@ def oozie_server_specific(upgrade_type):
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- if params.lzo_enabled and len(params.all_lzo_packages) > 0:
- Package(params.all_lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
- Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
- not_if = no_op_test,
- )
-
prepare_war(params)
File(hashcode_file,
@@ -363,7 +361,7 @@ def oozie_server_specific(upgrade_type):
Directory(params.oozie_server_dir,
owner = params.oozie_user,
group = params.user_group,
- recursive_ownership = True,
+ recursive_ownership = True,
)
if params.security_enabled:
File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
@@ -408,7 +406,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version)
if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version):
return
-
+
# Important that oozie_server_hostnames is sorted by name so that this only runs on a single Oozie server.
if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]):
Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first Oozie Server "
@@ -463,7 +461,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
recursive_chmod=True,
replace_existing_files=True
)
-
+
Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.")
atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename)
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index cd1f7ad..8f38843 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -31,7 +31,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.get_architecture import get_architecture
from resource_management.libraries.functions.stack_features import get_stack_feature_version
@@ -387,8 +387,4 @@ HdfsResource = functools.partial(
is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
-# The logic for LZO also exists in HDFS' params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-all_lzo_packages = get_lzo_packages(stack_version_unformatted)
+lzo_enabled = should_install_lzo()
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index f6e8391..197382c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script
from resource_management.core.resources.service import ServiceConfig
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import File
from resource_management.libraries.resources.xml_config import XmlConfig
@@ -110,6 +111,8 @@ def yarn(name=None, config_dir=None):
"""
import params
+ install_lzo_if_needed()
+
if config_dir is None:
config_dir = params.hadoop_conf_dir
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
index 07a9ef8..8aa69a9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
@@ -40,19 +40,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -74,19 +61,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -120,15 +94,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>hadooplzo-${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
index 63e85e3..e02bf6b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
@@ -72,19 +72,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -106,19 +93,6 @@
<name>snappy-devel</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadooplzo_${stack_version}-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -152,10 +126,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>hadooplzo-${stack_version}</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
index 86ca03a..29cbddc 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
@@ -55,7 +55,8 @@
"mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
"custom_mysql_jdbc_name" : "mysql-connector-java.jar",
"custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar",
- "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"
+ "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar",
+ "gpl_license_accepted": "true"
},
"commandType": "EXECUTION_COMMAND",
"roleParams": {