You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/11/14 23:37:56 UTC
ambari git commit: Revert "AMBARI-22383. Remove Auto-Installation of
LZO Libraries (ncole)"
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 62876f576 -> fe7c17941
Revert "AMBARI-22383. Remove Auto-Installation of LZO Libraries (ncole)"
This reverts commit 6df336124e3ad13c0793bd32b35931ded5219f2d.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fe7c1794
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fe7c1794
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fe7c1794
Branch: refs/heads/branch-2.6
Commit: fe7c17941b1544d10e5170fd031a3a9f4623e035
Parents: 62876f5
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Nov 14 18:37:43 2017 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Nov 14 18:37:43 2017 -0500
----------------------------------------------------------------------
.../libraries/functions/__init__.py | 1 +
.../libraries/functions/get_lzo_packages.py | 50 ++++++++++++++++++++
.../libraries/functions/package_conditions.py | 8 +++-
.../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 ++++++++++++
.../HDFS/2.1.0.2.0/package/scripts/datanode.py | 1 +
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 5 ++
.../2.1.0.2.0/package/scripts/hdfs_client.py | 1 +
.../2.1.0.2.0/package/scripts/install_params.py | 11 ++++-
.../2.1.0.2.0/package/scripts/journalnode.py | 1 +
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 1 +
.../2.1.0.2.0/package/scripts/params_linux.py | 6 +++
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 ++-
.../package/scripts/oozie_server_upgrade.py | 7 +--
.../4.0.0.2.0/package/scripts/params_linux.py | 5 +-
.../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 32 +++++++++++++
.../custom_actions/scripts/remove_bits.py | 2 +-
.../BIGTOP/0.8/services/HDFS/metainfo.xml | 16 +++++++
.../0.8/services/HDFS/package/scripts/params.py | 6 ++-
.../BigInsights/4.0/services/HDFS/metainfo.xml | 16 +++++++
.../4.0/services/HDFS/package/scripts/params.py | 5 ++
.../4.0/services/OOZIE/package/scripts/oozie.py | 5 ++
.../services/OOZIE/package/scripts/params.py | 19 ++++++++
.../BigInsights/4.1/services/HDFS/metainfo.xml | 13 +++++
.../4.2.5/services/HDFS/metainfo.xml | 12 +++++
.../4.2.5/services/HDFS/package/scripts/hdfs.py | 5 ++
.../HDFS/package/scripts/install_params.py | 7 +++
.../HDFS/package/scripts/params_linux.py | 6 +++
.../services/OOZIE/package/scripts/oozie.py | 5 +-
.../OOZIE/package/scripts/params_linux.py | 3 ++
.../BigInsights/4.2/services/HDFS/metainfo.xml | 20 ++++++++
.../4.2/services/HDFS/package/scripts/hdfs.py | 3 ++
.../HDFS/package/scripts/install_params.py | 8 ++++
.../4.2/services/HDFS/package/scripts/params.py | 9 +++-
.../4.2/services/OOZIE/package/scripts/oozie.py | 5 ++
.../services/OOZIE/package/scripts/params.py | 19 ++++++++
.../stacks/HDP/2.2/services/HDFS/metainfo.xml | 35 ++++++++++++++
.../2.3.GlusterFS/services/HDFS/metainfo.xml | 10 ++++
.../stacks/HDP/2.3/services/HDFS/metainfo.xml | 30 ++++++++++++
.../python/custom_actions/TestRemoveBits.py | 5 +-
39 files changed, 413 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index 1e388ac..f144b2d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -39,6 +39,7 @@ from resource_management.libraries.functions.version import *
from resource_management.libraries.functions.format_jvm_option import *
from resource_management.libraries.functions.constants import *
from resource_management.libraries.functions.get_stack_version import *
+from resource_management.libraries.functions.get_lzo_packages import *
from resource_management.libraries.functions.setup_ranger_plugin import *
from resource_management.libraries.functions.curl_krb_request import *
from resource_management.libraries.functions.get_bare_principal import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
new file mode 100644
index 0000000..cfbb7d8
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+__all__ = ["get_lzo_packages"]
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.script.script import Script
+
+# TODO: Make list of lzo packages stack driven
+def get_lzo_packages(stack_version_unformatted):
+ lzo_packages = []
+ script_instance = Script.get_instance()
+ if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
+ lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
+ elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
+ lzo_packages += ["lzo", "hadoop-lzo-native"]
+ elif OSCheck.is_ubuntu_family():
+ lzo_packages += ["liblzo2-2"]
+
+ if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
+ if OSCheck.is_ubuntu_family():
+ lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
+ script_instance.format_package_name("hadooplzo-${stack_version}-native")]
+ else:
+ lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
+ script_instance.format_package_name("hadooplzo_${stack_version}-native")]
+ else:
+ lzo_packages += ["hadoop-lzo"]
+
+ return lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index ded63cf..31e78b9 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -19,7 +19,7 @@ limitations under the License.
Ambari Agent
"""
-__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
+__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
"should_install_mysql", "should_install_ranger_tagsync"]
import os
@@ -44,6 +44,12 @@ def _has_local_components(config, components, indicator_function = any):
def _has_applicable_local_component(config, components):
return _has_local_components(config, components, any)
+def should_install_lzo():
+ config = Script.get_config()
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+ return lzo_enabled
+
def should_install_phoenix():
phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 2ab5ef8..2df1b9f 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -203,6 +203,11 @@
<package>
<name>hadoop</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
</packages>
</osSpecific>
@@ -219,6 +224,16 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -237,6 +252,16 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -255,6 +280,11 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py
index bbb823d..8bfa7e6 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/datanode.py
@@ -148,6 +148,7 @@ class DataNodeDefault(DataNode):
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class DataNodeWindows(DataNode):
def install(self, env):
+ import install_params
self.install_packages(env)
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index 3d6b7d5..6f702d3 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -133,6 +133,11 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
+ if params.lzo_enabled and len(params.lzo_packages) > 0:
+ Package(params.lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
+
def install_snappy():
import params
Directory([params.so_target_dir_x86, params.so_target_dir_x64],
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py
index fee0cb5..9c4f9d4 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_client.py
@@ -65,6 +65,7 @@ class HdfsClientDefault(HdfsClient):
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class HdfsClientWindows(HdfsClient):
def install(self, env):
+ import install_params
self.install_packages(env)
self.configure(env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
index 2f114a7..fe488c3 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
@@ -18,13 +18,22 @@ limitations under the License.
"""
from ambari_commons import OSCheck
-exclude_packages = []
# These parameters are supposed to be referenced at installation time, before the Hadoop environment variables have been set
if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
+ from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
+
+ # The logic for LZO also exists in OOZIE's params.py
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+ lzo_packages = get_lzo_packages(stack_version_unformatted)
+
+ exclude_packages = []
+ if not lzo_enabled:
+ exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py
index bd10289..25534e9 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py
@@ -110,6 +110,7 @@ class JournalNodeDefault(JournalNode):
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class JournalNodeWindows(JournalNode):
def install(self, env):
+ import install_params
self.install_packages(env)
def start(self, env):
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
index 95670fe..89719cf 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py
@@ -306,6 +306,7 @@ class NameNodeDefault(NameNode):
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class NameNodeWindows(NameNode):
def install(self, env):
+ import install_params
self.install_packages(env)
#TODO we need this for HA because of manual steps
self.configure(env)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 2888b78..5ca012b 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -40,6 +40,7 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -384,6 +385,11 @@ HdfsResource = functools.partial(
)
+# The logic for LZO also exists in OOZIE's params.py
+io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+lzo_packages = get_lzo_packages(stack_version_unformatted)
+
name_node_params = default("/commandParams/namenode", None)
java_home = config['hostLevelParams']['java_home']
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 50d9a4a..7d22603 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -305,8 +305,10 @@ def oozie_server_specific(upgrade_type):
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- # just copying files is ok - we're not making assumptions about installing LZO here
- if params.lzo_enabled:
+ if params.lzo_enabled and len(params.all_lzo_packages) > 0:
+ Package(params.all_lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index e07616c..eb57c22 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -61,15 +61,10 @@ class OozieUpgrade(Script):
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
- # for downgrade to 2.2.0.0 since hadoop-lzo will not be present.
- #
+ # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
- #
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
- #
- # This block is just copying around files - there is no assumption about installation
- #
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("{stack_root}/{version}/hadoop/lib")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index 7c198c4..cd1f7ad 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -31,6 +31,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.get_architecture import get_architecture
from resource_management.libraries.functions.stack_features import get_stack_feature_version
@@ -388,4 +389,6 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'
# The logic for LZO also exists in HDFS' params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
\ No newline at end of file
+lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+
+all_lzo_packages = get_lzo_packages(stack_version_unformatted)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
index dcf3e24..ed3f5fd 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
@@ -19,15 +19,22 @@ Ambari Agent
"""
import os
+import urlparse
from ambari_commons import OSConst
+from ambari_commons.inet_utils import download_file
from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons.os_utils import copy_file, extract_path_component
from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.get_stack_version import get_stack_version
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.default import default
+from resource_management.core.logger import Logger
from tez import tez
@@ -89,8 +96,33 @@ class TezClientWindows(TezClient):
self.install_packages(env)
params.refresh_tez_state_dependent_params()
env.set_params(params)
+ self._install_lzo_support_if_needed(params)
self.configure(env, config_dir=params.tez_conf_dir)
+ def _install_lzo_support_if_needed(self, params):
+ hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])
+
+ hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
+ if hadoop_lzo_dest_path:
+ hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
+
+ config = Script.get_config()
+ file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
+ hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
+ download_file(file_url, hadoop_lzo_dl_path)
+ #This is for protection against configuration changes. It will infect every new destination with the lzo jar,
+ # but since the classpath points to the jar directly we're getting away with it.
+ if not os.path.exists(hadoop_lzo_dest_path):
+ copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
+
+ def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template, configurations):
+ import resource_management
+
+ hadoop_classpath_prefix_obj = InlineTemplate(hadoop_classpath_prefix_template, configurations_dict=configurations,
+ extra_imports=[resource_management, resource_management.core,
+ resource_management.core.source])
+ hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content()
+ return hadoop_classpath_prefix
if __name__ == "__main__":
TezClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
index 078cf32..0be9a34 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_bits.py
@@ -43,7 +43,7 @@ class RemoveBits(Script):
Logger.info("Attempting to remove bits for HDP 2.1")
config = Script.get_config()
- packages_to_remove = ["zookeeper", "hadoop", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"]
+ packages_to_remove = ["zookeeper", "hadoop", "hadoop-lzo", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"]
packages_to_remove.reverse()
Logger.info("Packages to remove: {0}".format(" ".join(packages_to_remove)))
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml
index f473598..8500839 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/metainfo.xml
@@ -180,6 +180,10 @@
<package>
<name>hadoop</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
</packages>
</osSpecific>
@@ -193,6 +197,14 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
<name>hadoop-libhdfs</name>
</package>
</packages>
@@ -208,6 +220,10 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
index f16242c..3a711c7 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py
@@ -209,7 +209,11 @@ HdfsDirectory = functools.partial(
bin_dir = hadoop_bin_dir
)
-exclude_packages = []
+io_compression_codecs = config['configurations']['core-site']['io.compression.codecs']
+if not "com.hadoop.compression.lzo" in io_compression_codecs:
+ exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"]
+else:
+ exclude_packages = []
name_node_params = default("/commandParams/namenode", None)
#hadoop params
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
index 1efd746..918cdb3 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
@@ -152,6 +152,10 @@
<package>
<name>hadoop</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ <condition>should_install_lzo</condition>
+ </package>
</packages>
</osSpecific>
@@ -162,6 +166,14 @@
<name>snappy</name>
</package>
<package>
+ <name>lzo</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-libhdfs</name>
</package>
<package>
@@ -180,6 +192,10 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py
index d604906..1f76f80 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/package/scripts/params.py
@@ -293,6 +293,11 @@ HdfsResource = functools.partial(
default_fs = default_fs
)
+io_compression_codecs = config['configurations']['core-site']['io.compression.codecs']
+if not "com.hadoop.compression.lzo" in io_compression_codecs:
+ exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"]
+else:
+ exclude_packages = []
name_node_params = default("/commandParams/namenode", None)
#hadoop params
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py
index 76a2abe..6cd2b71 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/oozie.py
@@ -227,6 +227,11 @@ def oozie_server_specific():
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'),
not_if = no_op_test,
)
+ #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0:
+ # Package(params.lzo_packages_for_current_host)
+ # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ # not_if = no_op_test,
+ # )
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py
index 17ea124..febd3e6 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/OOZIE/package/scripts/params.py
@@ -235,6 +235,25 @@ HdfsResource = functools.partial(
#-----LZO is not suppported in IOP distribution since it is GPL license--------
+'''
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+# stack_is_iop40_or_further
+underscored_version = stack_version_unformatted.replace('.', '_')
+dashed_version = stack_version_unformatted.replace('.', '-')
+lzo_packages_to_family = {
+ "any": ["hadoop-lzo", ],
+ "redhat": ["lzo", "hadoop-lzo-native"],
+ "suse": ["lzo", "hadoop-lzo-native"],
+ "ubuntu": ["liblzo2-2", ]
+}
+
+
+lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")]
+lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")]
+lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")]
+
+lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family]
+all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values()))
+'''
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml
index 45b16ee..4c00b4f 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/HDFS/metainfo.xml
@@ -62,6 +62,10 @@
<package>
<name>hadoop_4_1_*</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ <!--name>hadoop_4_1_*-lzo</name-->
+ </package>
</packages>
</osSpecific>
@@ -72,6 +76,12 @@
<name>snappy</name>
</package>
<package>
+ <name>lzo</name>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ </package>
+ <package>
<name>hadoop_4_1_*-libhdfs</name>
</package>
<package>
@@ -90,6 +100,9 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml
index 95724b7..fc7c324 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/metainfo.xml
@@ -63,6 +63,9 @@
<package>
<name>hadoop_4_2_5_*</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -72,6 +75,12 @@
<name>snappy</name>
</package>
<package>
+ <name>lzo</name>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ </package>
+ <package>
<name>hadoop_4_2_5_*-libhdfs</name>
</package>
<package>
@@ -95,6 +104,9 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py
index 096c4d7..1264284 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/hdfs.py
@@ -116,6 +116,11 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
+ if params.lzo_enabled and len(params.lzo_packages) > 0:
+ Package(params.lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
+
def install_snappy():
import params
Directory([params.so_target_dir_x86, params.so_target_dir_x64],
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py
index 9d2c4f3..da8de46 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/install_params.py
@@ -28,4 +28,11 @@ else:
_config = Script.get_config()
stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
+ # The logic for LZO also exists in OOZIE's params.py
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+ lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"]
+
exclude_packages = []
+ if not lzo_enabled:
+ exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py
index 37c2b85..a77ec82 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py
@@ -362,6 +362,12 @@ HdfsResource = functools.partial(
dfs_type = dfs_type
)
+
+# The logic for LZO also exists in OOZIE's params.py
+io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"]
+
name_node_params = default("/commandParams/namenode", None)
java_home = config['hostLevelParams']['java_home']
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py
index 5f572a7..33a309f 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py
@@ -305,7 +305,10 @@ def oozie_server_specific(upgrade_type):
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- if params.lzo_enabled:
+ if params.lzo_enabled and len(params.all_lzo_packages) > 0:
+ Package(params.all_lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp /usr/lib/hadoop-lzo/lib/hadoop-lzo*.jar {oozie_lib_dir}'),
not_if = no_op_test,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py
index f87e3db..0e6179b 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/params_linux.py
@@ -30,6 +30,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.get_architecture import get_architecture
from resource_management.libraries.functions.stack_features import get_stack_feature_version
@@ -382,3 +383,5 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'
# The logic for LZO also exists in HDFS' params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+
+all_lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
index 4253eb1..72ce2be 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
@@ -182,6 +182,11 @@
<package>
<name>hadoop_4_2_*</name>
</package>
+ <package>
+ <name>hadoop-lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -191,6 +196,16 @@
<name>snappy</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadoop-lzo-native</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop_4_2_*-libhdfs</name>
</package>
<package>
@@ -214,6 +229,11 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
index cf54f1f..2b158bd 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
@@ -115,6 +115,9 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
+ if params.lzo_enabled and len(params.lzo_packages) > 0:
+ Package(params.lzo_packages)
+
def install_snappy():
import params
Directory([params.so_target_dir_x86, params.so_target_dir_x64],
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py
index 9d2c4f3..fe488c3 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/install_params.py
@@ -23,9 +23,17 @@ if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
+ from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
+ # The logic for LZO also exists in OOZIE's params.py
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+ lzo_packages = get_lzo_packages(stack_version_unformatted)
+
exclude_packages = []
+ if not lzo_enabled:
+ exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py
index cd426b3..7b9c449 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/params.py
@@ -304,7 +304,14 @@ HdfsResource = functools.partial(
default_fs = default_fs
)
-
+lzo_enabled = True
+lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"]
+io_compression_codecs = config['configurations']['core-site']['io.compression.codecs']
+if not "com.hadoop.compression.lzo" in io_compression_codecs:
+ lzo_enabled = False
+ exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"]
+else:
+ exclude_packages = []
name_node_params = default("/commandParams/namenode", '{"threshold":"10"}')
#hadoop params
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
index ac69d37..64a83c0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
@@ -227,6 +227,11 @@ def oozie_server_specific():
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'),
not_if = no_op_test,
)
+ #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0:
+ # Package(params.lzo_packages_for_current_host)
+ # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ # not_if = no_op_test,
+ # )
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
index a3b0c3f..d1a4a66 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
@@ -238,6 +238,25 @@ HdfsResource = functools.partial(
#-----LZO is not suppported in IOP distribution since it is GPL license--------
+'''
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+# stack_is_iop40_or_further
+underscored_version = stack_version_unformatted.replace('.', '_')
+dashed_version = stack_version_unformatted.replace('.', '-')
+lzo_packages_to_family = {
+ "any": ["hadoop-lzo", ],
+ "redhat": ["lzo", "hadoop-lzo-native"],
+ "suse": ["lzo", "hadoop-lzo-native"],
+ "ubuntu": ["liblzo2-2", ]
+}
+
+
+lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")]
+lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")]
+lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")]
+
+lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family]
+all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values()))
+'''
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
index 8aa69a9..07a9ef8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
@@ -40,6 +40,19 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -61,6 +74,19 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -94,6 +120,15 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>hadooplzo-${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml
index 6a9fab61..15fe931 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HDFS/metainfo.xml
@@ -36,6 +36,13 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>hadooplzo_2_3_*</name>
+ </package>
+ <package>
<name>hadoop_2_3_*-libhdfs</name>
</package>
</packages>
@@ -69,6 +76,9 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>hadooplzo-2-3-.*</name>
+ </package>
+ <package>
<name>libhdfs0-2-3-.*</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
index e02bf6b..63e85e3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
@@ -72,6 +72,19 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>lzo</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -93,6 +106,19 @@
<name>snappy-devel</name>
</package>
<package>
+ <name>liblzo2-2</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
+ <name>hadooplzo_${stack_version}-native</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>hadoop_${stack_version}-libhdfs</name>
</package>
</packages>
@@ -126,6 +152,10 @@
<name>libsnappy-dev</name>
</package>
<package>
+ <name>hadooplzo-${stack_version}</name>
+ <condition>should_install_lzo</condition>
+ </package>
+ <package>
<name>libhdfs0-${stack_version}</name>
</package>
</packages>
http://git-wip-us.apache.org/repos/asf/ambari/blob/fe7c1794/ambari-server/src/test/python/custom_actions/TestRemoveBits.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveBits.py b/ambari-server/src/test/python/custom_actions/TestRemoveBits.py
index 8217ca0..6537152 100644
--- a/ambari-server/src/test/python/custom_actions/TestRemoveBits.py
+++ b/ambari-server/src/test/python/custom_actions/TestRemoveBits.py
@@ -105,10 +105,13 @@ class TestRemoveBits(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop-hdfs',
action = ['remove'],
)
+ self.assertResourceCalled('Package', 'hadoop-lzo',
+ action = ['remove'],
+ )
self.assertResourceCalled('Package', 'hadoop',
action = ['remove'],
)
self.assertResourceCalled('Package', 'zookeeper',
action = ['remove'],
)
- self.assertNoMoreResources()
+ self.assertNoMoreResources()
\ No newline at end of file