You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/10/11 05:25:18 UTC
[22/31] ambari git commit: AMBARI-22160. hadooplzo package
installation failed on devdeploys (aonishuk)
AMBARI-22160. hadooplzo package installation failed on devdeploys (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e19db403
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e19db403
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e19db403
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: e19db403e8a3320a5b47503e43e2a35277149a3f
Parents: e037a8d
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Oct 10 15:32:10 2017 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Oct 10 15:32:10 2017 +0300
----------------------------------------------------------------------
.../python/resource_management/libraries/script/script.py | 4 ++--
.../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 10 ++++++----
.../HDFS/2.1.0.2.0/package/scripts/install_params.py | 6 ------
.../HDFS/2.1.0.2.0/package/scripts/params_linux.py | 2 --
.../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 10 ++++++----
.../HDFS/3.0.0.3.0/package/scripts/install_params.py | 6 ------
.../HDFS/3.0.0.3.0/package/scripts/params_linux.py | 2 --
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 ++++--
.../OOZIE/4.0.0.2.0/package/scripts/params_linux.py | 3 ---
.../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 5 +++--
.../OOZIE/4.2.0.3.0/package/scripts/params_linux.py | 3 ---
11 files changed, 21 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 4282213..bf8c0dc 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -778,10 +778,10 @@ class Script(object):
pkg_provider = get_provider("Package")
try:
- available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories'])
+ self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories'])
except Exception as err:
Logger.exception("Unable to load available packages")
- available_packages_in_repos = []
+ self.available_packages_in_repos = []
def install_packages(self, env):
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index e054209..07c7616 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -25,6 +25,7 @@ from resource_management.core.resources import Package
from resource_management.core.source import Template
from resource_management.core.resources.service import ServiceConfig
from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
import os
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
@@ -138,10 +139,11 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
+ if params.lzo_enabled:
+ lzo_packages = get_lzo_packages(params.stack_version_unformatted)
+ Package(lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
index fe488c3..235f231 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
@@ -23,7 +23,6 @@ if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
- from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
@@ -32,8 +31,3 @@ else:
# The logic for LZO also exists in OOZIE's params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- lzo_packages = get_lzo_packages(stack_version_unformatted)
-
- exclude_packages = []
- if not lzo_enabled:
- exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 76b430b..bb6349b 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -389,7 +388,6 @@ HdfsResource = functools.partial(
# The logic for LZO also exists in OOZIE's params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
name_node_params = default("/commandParams/namenode", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
index e054209..07c7616 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/hdfs.py
@@ -25,6 +25,7 @@ from resource_management.core.resources import Package
from resource_management.core.source import Template
from resource_management.core.resources.service import ServiceConfig
from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
import os
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
@@ -138,10 +139,11 @@ def hdfs(name=None):
content=Template("slaves.j2")
)
- if params.lzo_enabled and len(params.lzo_packages) > 0:
- Package(params.lzo_packages,
- retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
- retry_count=params.agent_stack_retry_count)
+ if params.lzo_enabled:
+ lzo_packages = get_lzo_packages(params.stack_version_unformatted)
+ Package(lzo_packages,
+ retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
+ retry_count=params.agent_stack_retry_count)
def install_snappy():
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py
index fe488c3..235f231 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/install_params.py
@@ -23,7 +23,6 @@ if OSCheck.is_windows_family():
exclude_packages = []
else:
from resource_management.libraries.functions.default import default
- from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.script.script import Script
_config = Script.get_config()
@@ -32,8 +31,3 @@ else:
# The logic for LZO also exists in OOZIE's params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
- lzo_packages = get_lzo_packages(stack_version_unformatted)
-
- exclude_packages = []
- if not lzo_enabled:
- exclude_packages += lzo_packages
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
index de735f4..2fa6208 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import ge
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.format_jvm_option import format_jvm_option
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config
@@ -378,7 +377,6 @@ HdfsResource = functools.partial(
# The logic for LZO also exists in OOZIE's params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
name_node_params = default("/commandParams/namenode", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 64f9d54..f215a1e 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -37,6 +37,7 @@ from resource_management.libraries.functions.copy_tarball import get_current_ver
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.security_commons import update_credential_provider_path
+from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.core.resources.packaging import Package
from resource_management.core.shell import as_user, as_sudo, call, checked_call
from resource_management.core.exceptions import Fail
@@ -305,8 +306,9 @@ def oozie_server_specific(upgrade_type):
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- if params.lzo_enabled and len(params.all_lzo_packages) > 0:
- Package(params.all_lzo_packages,
+ if params.lzo_enabled:
+ all_lzo_packages = get_lzo_packages(params.stack_version_unformatted)
+ Package(all_lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index b66e157..a0f0672 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -30,7 +30,6 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -388,5 +387,3 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'
# The logic for LZO also exists in HDFS' params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-all_lzo_packages = get_lzo_packages(stack_version_unformatted)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
index d916d3b..0771e93 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
@@ -275,8 +275,9 @@ def oozie_server_specific():
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
not_if = no_op_test)
- if params.lzo_enabled and len(params.all_lzo_packages) > 0:
- Package(params.all_lzo_packages,
+ if params.lzo_enabled:
+ all_lzo_packages = get_lzo_packages(params.stack_version_unformatted)
+ Package(all_lzo_packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/e19db403/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py
index d30a465..70b89b7 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/params_linux.py
@@ -28,7 +28,6 @@ from resource_management.libraries.functions import get_port_from_url
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.get_architecture import get_architecture
@@ -370,5 +369,3 @@ is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'
# The logic for LZO also exists in HDFS' params.py
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-all_lzo_packages = get_lzo_packages(stack_version_unformatted)