You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/11/28 21:20:33 UTC
[14/24] ambari git commit: AMBARI-22489. Port AMBARI-22467 to
BigInsight (aonishuk)
AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aea1a3c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aea1a3c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aea1a3c4
Branch: refs/heads/branch-2.6
Commit: aea1a3c48fdf58be5fad04d7b983c65d28a248f9
Parents: c7457a8
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Nov 22 12:32:04 2017 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Nov 22 12:32:04 2017 +0200
----------------------------------------------------------------------
.../BigInsights/4.0/services/HDFS/metainfo.xml | 17 ------------
.../BigInsights/4.2/services/HDFS/metainfo.xml | 20 --------------
.../4.2/services/HDFS/package/scripts/hdfs.py | 5 ++--
.../4.2/services/HIVE/package/scripts/hive.py | 3 +++
.../4.2/services/OOZIE/package/scripts/oozie.py | 12 +++++----
.../services/OOZIE/package/scripts/params.py | 28 ++------------------
.../4.2/services/YARN/package/scripts/yarn.py | 3 +++
7 files changed, 17 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
index 918cdb3..82c82d0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
@@ -152,10 +152,6 @@
<package>
<name>hadoop</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
@@ -165,15 +161,6 @@
<package>
<name>snappy</name>
</package>
- <package>
- <name>lzo</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-libhdfs</name>
</package>
<package>
@@ -192,10 +179,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
index 72ce2be..4253eb1 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
@@ -182,11 +182,6 @@
<package>
<name>hadoop_4_2_*</name>
</package>
- <package>
- <name>hadoop-lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
</packages>
</osSpecific>
<osSpecific>
@@ -196,16 +191,6 @@
<name>snappy</name>
</package>
<package>
- <name>lzo</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
- <name>hadoop-lzo-native</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop_4_2_*-libhdfs</name>
</package>
<package>
@@ -229,11 +214,6 @@
<name>libsnappy-dev</name>
</package>
<package>
- <name>liblzo2-2</name>
- <skipUpgrade>true</skipUpgrade>
- <condition>should_install_lzo</condition>
- </package>
- <package>
<name>hadoop-hdfs</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
index 2b158bd..9334de0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
@@ -20,10 +20,10 @@ Ambari Agent
"""
from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
import sys
import os
-
def hdfs(name=None):
import params
@@ -115,8 +115,7 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages)
+ install_lzo_if_needed()
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
index 0446e1c..f451a65 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
@@ -34,6 +34,7 @@ from resource_management.core.shell import as_user
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.core.exceptions import Fail
from resource_management.core.shell import as_sudo
from resource_management.core.shell import quote_bash_args
@@ -48,6 +49,8 @@ from ambari_commons import OSConst
def hive(name=None):
import params
+ install_lzo_if_needed()
+
XmlConfig("hive-site.xml",
conf_dir = params.hive_conf_dir,
configurations = params.config['configurations']['hive-site'],
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
index 64a83c0..0459432 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
@@ -26,6 +26,7 @@ from resource_management.core.resources.system import Execute
from resource_management.core.source import DownloadSource
from resource_management.core.source import InlineTemplate
from resource_management.core.source import Template
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
from resource_management.libraries.functions import format
from resource_management.libraries.functions import compare_versions
from resource_management.libraries.resources.xml_config import XmlConfig
@@ -99,6 +100,12 @@ def oozie(is_server=False):
pass
oozie_ownership()
+
+ if params.lzo_enabled:
+ install_lzo_if_needed()
+ Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+ not_if = no_op_test,
+ )
if is_server:
oozie_server_specific()
@@ -227,11 +234,6 @@ def oozie_server_specific():
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'),
not_if = no_op_test,
)
- #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0:
- # Package(params.lzo_packages_for_current_host)
- # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
- # not_if = no_op_test,
- # )
prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
index d1a4a66..ee9b88a 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
@@ -27,6 +27,7 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
from resource_management.libraries.resources import HdfsResource
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
@@ -234,29 +235,4 @@ HdfsResource = functools.partial(
default_fs = default_fs
)
-#LZO support
-
-#-----LZO is not suppported in IOP distribution since it is GPL license--------
-
-'''
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-# stack_is_iop40_or_further
-underscored_version = stack_version_unformatted.replace('.', '_')
-dashed_version = stack_version_unformatted.replace('.', '-')
-lzo_packages_to_family = {
- "any": ["hadoop-lzo", ],
- "redhat": ["lzo", "hadoop-lzo-native"],
- "suse": ["lzo", "hadoop-lzo-native"],
- "ubuntu": ["liblzo2-2", ]
-}
-
-
-lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")]
-
-lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family]
-all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values()))
-'''
+lzo_enabled = should_install_lzo()
http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
index a284183..9006d2e 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
@@ -20,6 +20,7 @@ Ambari Agent
"""
from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
import sys
import os
@@ -50,6 +51,8 @@ def create_local_dir(dir_name):
def yarn(name = None):
import params
+
+ install_lzo_if_needed()
if name == "historyserver":
if params.yarn_log_aggregation_enabled: