You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/12/06 15:06:25 UTC
[09/14] ambari git commit: AMBARI-22598 - Pig service check failed
after PU with LzoCodec CNF (jonathanhurley)
AMBARI-22598 - Pig service check failed after PU with LzoCodec CNF (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/508eba0c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/508eba0c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/508eba0c
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 508eba0c62aa42cea5bcc660a4661ff373a5af91
Parents: 97ceed0
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Tue Dec 5 13:30:55 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Dec 6 08:59:14 2017 -0500
----------------------------------------------------------------------
.../MAHOUT/1.0.0.2.3/package/scripts/mahout.py | 12 +++++++---
.../PIG/0.12.0.2.0/package/scripts/pig.py | 16 +++++++++----
.../PIG/0.16.1.3.0/package/scripts/pig.py | 16 +++++++++----
.../SPARK/1.2.1/package/scripts/setup_spark.py | 20 ++++++++--------
.../SPARK/2.2.0/package/scripts/setup_spark.py | 25 ++++++++++----------
.../SPARK2/2.0.0/package/scripts/setup_spark.py | 22 ++++++++---------
6 files changed, 65 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py
index 68d6844..d2df928 100644
--- a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py
+++ b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py
@@ -20,14 +20,20 @@ Ambari Agent
"""
import os
-from resource_management.core.resources.system import Directory, File
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.functions.format import format
+from resource_management.core.resources import Directory
+from resource_management.core.resources import File
+from resource_management.libraries.functions import format
from resource_management.libraries.functions import is_empty
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import XmlConfig
+
def mahout():
import params
+ # ensure that matching LZO libraries are installed for Mahout
+ lzo_utils.install_lzo_if_needed()
+
Directory( params.mahout_conf_dir,
create_parents = True,
owner = params.mahout_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
index b6825d5..1378fdc 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
@@ -19,16 +19,22 @@ Ambari Agent
"""
import os
-from resource_management.core.resources.system import Directory, File
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions.format import format
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.core.resources import Directory
+from resource_management.core.resources import File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def pig():
import params
+ # ensure that matching LZO libraries are installed for Pig
+ lzo_utils.install_lzo_if_needed()
+
Directory( params.pig_conf_dir,
create_parents = True,
owner = params.hdfs_user,
@@ -49,7 +55,7 @@ def pig():
content=params.pig_properties
)
- if (params.log4j_props != None):
+ if (params.log4j_props is not None):
File(format("{params.pig_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
@@ -72,7 +78,7 @@ def pig():
content=params.pig_properties
)
- if (params.log4j_props != None):
+ if (params.log4j_props is not None):
File(os.path.join(params.pig_conf_dir, "log4j.properties"),
mode='f',
owner=params.pig_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
index b6825d5..1378fdc 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
@@ -19,16 +19,22 @@ Ambari Agent
"""
import os
-from resource_management.core.resources.system import Directory, File
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions.format import format
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.core.resources import Directory
+from resource_management.core.resources import File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def pig():
import params
+ # ensure that matching LZO libraries are installed for Pig
+ lzo_utils.install_lzo_if_needed()
+
Directory( params.pig_conf_dir,
create_parents = True,
owner = params.hdfs_user,
@@ -49,7 +55,7 @@ def pig():
content=params.pig_properties
)
- if (params.log4j_props != None):
+ if (params.log4j_props is not None):
File(format("{params.pig_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
@@ -72,7 +78,7 @@ def pig():
content=params.pig_properties
)
- if (params.log4j_props != None):
+ if (params.log4j_props is not None):
File(os.path.join(params.pig_conf_dir, "log4j.properties"),
mode='f',
owner=params.pig_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
index 6a29efb..830a7ee 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
@@ -18,21 +18,18 @@ limitations under the License.
"""
-import sys
-import fileinput
-import shutil
import os
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core import shell
+
from resource_management.core.source import InlineTemplate
from resource_management.core.resources.system import Directory, File
-from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.constants import StackFeature
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import PropertiesFile
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.resources import XmlConfig
+
def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
"""
@@ -45,6 +42,9 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
import params
+ # ensure that matching LZO libraries are installed for Spark
+ lzo_utils.install_lzo_if_needed()
+
if config_dir is None:
config_dir = params.spark_conf
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py
index 9329ce0..148a967 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py
@@ -18,31 +18,32 @@ limitations under the License.
"""
-import sys
-import fileinput
-import shutil
import os
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core import shell
+from resource_management.core.resources import Directory
+from resource_management.core.resources import File
from resource_management.core.source import InlineTemplate
-from resource_management.core.resources.system import Directory, File
-from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.constants import StackFeature
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import PropertiesFile
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.resources import XmlConfig
def setup_spark(env, type, upgrade_type = None, action = None):
import params
+ # ensure that matching LZO libraries are installed for Spark
+ lzo_utils.install_lzo_if_needed()
+
Directory([params.spark_pid_dir, params.spark_log_dir],
owner=params.spark_user,
group=params.user_group,
mode=0775,
- create_parents = True
+ create_parents = True,
+ cd_access = 'a',
)
if type == 'server' and action == 'config':
params.HdfsResource(params.spark_hdfs_user_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py
index 792b2a1..3638953 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py
@@ -18,26 +18,26 @@ limitations under the License.
"""
-import sys
-import fileinput
-import shutil
import os
-from resource_management.core.exceptions import ComponentIsNotRunning
-from resource_management.core.logger import Logger
-from resource_management.core import shell
+from resource_management.core.resources import Directory
+from resource_management.core.resources import File
from resource_management.core.source import InlineTemplate
-from resource_management.core.resources.system import Directory, File
-from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.constants import StackFeature
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import lzo_utils
+from resource_management.libraries.resources import PropertiesFile
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.resources import HdfsResource
+from resource_management.libraries.resources import XmlConfig
def setup_spark(env, type, upgrade_type = None, action = None):
import params
+ # ensure that matching LZO libraries are installed for Spark
+ lzo_utils.install_lzo_if_needed()
+
Directory([params.spark_pid_dir, params.spark_log_dir],
owner=params.spark_user,
group=params.user_group,