You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/02/19 02:43:49 UTC
ambari git commit: AMBARI-15067: Add custom commands for HAWQ to run
check and clear cache(mithmatt via jaoki)
Repository: ambari
Updated Branches:
refs/heads/branch-2.2 86d9c1187 -> 1e93caf4f
AMBARI-15067: Add custom commands for HAWQ to run check and clear cache(mithmatt via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e93caf4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e93caf4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e93caf4
Branch: refs/heads/branch-2.2
Commit: 1e93caf4f7ba92f0190433303b7920eaf54c57eb
Parents: 86d9c11
Author: Jun Aoki <ja...@apache.org>
Authored: Thu Feb 18 17:43:36 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Thu Feb 18 17:43:36 2016 -0800
----------------------------------------------------------------------
.../common-services/HAWQ/2.0.0/metainfo.xml | 16 ++++++++++
.../2.0.0/package/scripts/hawq_constants.py | 2 +-
.../HAWQ/2.0.0/package/scripts/hawqmaster.py | 19 +++++++++++
.../HAWQ/2.0.0/package/scripts/master_helper.py | 16 +++++-----
.../HAWQ/2.0.0/package/scripts/params.py | 6 ++++
.../HAWQ/2.0.0/package/templates/hawq-hosts.j2 | 22 -------------
.../HAWQ/2.0.0/package/templates/slaves.j2 | 22 -------------
ambari-web/app/messages.js | 2 ++
ambari-web/app/models/host_component.js | 33 +++++++++++++++-----
ambari-web/app/utils/helper.js | 20 +++++++-----
ambari-web/app/views/main/service/item.js | 3 +-
11 files changed, 90 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
index 942b250..4078990 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
@@ -46,6 +46,22 @@
<timeout>1200</timeout>
</commandScript>
</customCommand>
+ <customCommand>
+ <name>HAWQ_CLEAR_CACHE</name>
+ <commandScript>
+ <script>scripts/hawqmaster.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>RUN_HAWQ_CHECK</name>
+ <commandScript>
+ <script>scripts/hawqmaster.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </customCommand>
</customCommands>
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py
index 9fdbde5..2a671da 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py
@@ -47,7 +47,7 @@ sysctl_conf_dir = "/etc/sysctl.d"
# Files
hawq_slaves_file = os.path.join(hawq_config_dir, "slaves")
hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh")
-hawq_hosts_file = "/tmp/hawq_hosts"
+hawq_hosts_file = os.path.join(hawq_config_dir, "hawq_hosts")
hawq_check_file = os.path.join(hawq_config_dir, "hawq_check.cnf")
sysctl_suse_file = "/etc/sysctl.conf"
sysctl_backup_file = "/etc/sysctl.conf.backup.{0}"
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
index d2f9ad0..3309561 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
@@ -17,7 +17,13 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
from resource_management import Script
+from resource_management.core.resources.system import Execute
+from resource_management.core.logger import Logger
from resource_management.libraries.functions.check_process_status import check_process_status
+try:
+ from resource_management.libraries.functions import hdp_select as hadoop_select
+except ImportError:
+ from resource_management.libraries.functions import phd_select as hadoop_select
import master_helper
import common
@@ -54,5 +60,18 @@ class HawqMaster(Script):
def immediate_stop_hawq_service(self, env):
master_helper.stop(hawq_constants.IMMEDIATE, hawq_constants.CLUSTER)
+ def hawq_clear_cache(self, env):
+ import params
+ from utils import exec_psql_cmd
+ cmd = "SELECT gp_metadata_cache_clear()"
+ Logger.info("Clearing HAWQ's HDFS Metadata cache ...")
+ exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port)
+
+ def run_hawq_check(self, env):
+ Logger.info("Executing HAWQ Check ...")
+ Execute("source {0} && hawq check -f {1} --hadoop {2} --config {3}".format(hawq_constants.hawq_greenplum_path_file, hawq_constants.hawq_hosts_file, hadoop_select.get_hadoop_dir('home'), hawq_constants.hawq_check_file),
+ user=hawq_constants.hawq_user,
+ timeout=hawq_constants.default_exec_timeout)
+
if __name__ == "__main__":
HawqMaster().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
index c5f891b..9482b35 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
@@ -18,7 +18,7 @@ limitations under the License.
import os
import sys
from resource_management.core.resources.system import File, Execute
-from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.libraries.functions.format import format
@@ -34,14 +34,14 @@ def __setup_master_specific_conf_files():
"""
import params
- File(hawq_constants.hawq_check_file, content=params.hawq_check_content, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
- mode=0644)
+ params.File(hawq_constants.hawq_check_file,
+ content=params.hawq_check_content)
- File(hawq_constants.hawq_slaves_file, content=Template("slaves.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
- mode=0644)
+ params.File(hawq_constants.hawq_slaves_file,
+ content=InlineTemplate("{% for host in hawqsegment_hosts %}{{host}}\n{% endfor %}"))
- File(hawq_constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
- mode=0644)
+ params.File(hawq_constants.hawq_hosts_file,
+ content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"))
def __setup_passwordless_ssh():
@@ -51,8 +51,6 @@ def __setup_passwordless_ssh():
import params
utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password))
- File(hawq_constants.hawq_hosts_file, action='delete')
-
def configure_master():
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
index 74c9813..7633731 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
@@ -19,6 +19,7 @@ limitations under the License.
import functools
import hawq_constants
from resource_management import Script
+from resource_management.core.resources.system import File
from resource_management.libraries.functions.default import default
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.resources.xml_config import XmlConfig
@@ -75,6 +76,11 @@ HdfsResource = functools.partial(HdfsResource,
hdfs_site=hdfs_site,
default_fs=default_fs)
+# File partial function
+File = functools.partial(File,
+ owner=hawq_constants.hawq_user,
+ group=hawq_constants.hawq_group,
+ mode=0644)
# XMLConfig partial function
XmlConfig = functools.partial(XmlConfig,
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
deleted file mode 100644
index e093375..0000000
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{% for host in hawq_all_hosts %}
-{{host}}
-{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
deleted file mode 100644
index 0435b3e..0000000
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{% for host in hawqsegment_hosts %}
-{{host}}
-{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index a7e4ff3..2d47606 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1681,6 +1681,8 @@ Em.I18n.translations = {
'services.service.actions.run.immediateStopHawqSegment.context':'Stop HAWQ Segment (Immediate Mode)',
'services.service.actions.run.resyncHawqStandby.context':'Re-Sync HAWQ Standby',
'services.service.actions.run.resyncHawqStandby.label':'Re-Synchronize HAWQ Standby',
+ 'services.service.actions.run.clearHawqCache.label':'Clear HAWQ\'s HDFS Metadata Cache',
+ 'services.service.actions.run.runHawqCheck.label':'Run HAWQ Check',
'services.service.actions.manage_configuration_groups.short':'Manage Config Groups',
'services.service.actions.serviceActions':'Service Actions',
'services.service.summary.unknown':'unknown',
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-web/app/models/host_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js
index 5d919b9..3e06684 100644
--- a/ambari-web/app/models/host_component.js
+++ b/ambari-web/app/models/host_component.js
@@ -375,7 +375,6 @@ App.HostComponentActionMap = {
label: Em.I18n.t('services.service.actions.run.immediateStopHawqService.label'),
cssClass: 'icon-stop',
disabled: !HM || HM.get('workStatus') != App.HostComponentStatus.started
-
},
IMMEDIATE_STOP_HAWQ_SEGMENT: {
customCommand: 'IMMEDIATE_STOP_HAWQ_SEGMENT',
@@ -392,12 +391,6 @@ App.HostComponentActionMap = {
isHidden : App.get('isSingleNode') || !HS ,
disabled: !((!!HMComponent && HMComponent.get('startedCount') === 1) && (!!HS && HS.get('workStatus') === App.HostComponentStatus.started))
},
- MASTER_CUSTOM_COMMAND: {
- action: 'executeCustomCommand',
- cssClass: 'icon-play-circle',
- isHidden: false,
- disabled: false
- },
TOGGLE_ADD_HAWQ_STANDBY: {
action: 'addHawqStandby',
label: Em.I18n.t('admin.addHawqStandby.button.enable'),
@@ -411,7 +404,31 @@ App.HostComponentActionMap = {
label: Em.I18n.t('admin.activateHawqStandby.button.enable'),
cssClass: 'icon-arrow-up',
isHidden: App.get('isSingleNode') || !HS,
- disabled: false
+ disabled: false
+ },
+ HAWQ_CLEAR_CACHE: {
+ action: 'executeHawqCustomCommand',
+ customCommand: 'HAWQ_CLEAR_CACHE',
+ context: Em.I18n.t('services.service.actions.run.clearHawqCache.label'),
+ label: Em.I18n.t('services.service.actions.run.clearHawqCache.label'),
+ cssClass: 'icon-refresh',
+ isHidden : false,
+ disabled: !HM || HM.get('workStatus') != App.HostComponentStatus.started
+ },
+ RUN_HAWQ_CHECK: {
+ action: 'executeHawqCustomCommand',
+ customCommand: 'RUN_HAWQ_CHECK',
+ context: Em.I18n.t('services.service.actions.run.runHawqCheck.label'),
+ label: Em.I18n.t('services.service.actions.run.runHawqCheck.label'),
+ cssClass: 'icon-thumbs-up-alt',
+ isHidden : false,
+ disabled: false
+ },
+ MASTER_CUSTOM_COMMAND: {
+ action: 'executeCustomCommand',
+ cssClass: 'icon-play-circle',
+ isHidden: false,
+ disabled: false
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-web/app/utils/helper.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/utils/helper.js b/ambari-web/app/utils/helper.js
index 8ac754a..ea269e8 100644
--- a/ambari-web/app/utils/helper.js
+++ b/ambari-web/app/utils/helper.js
@@ -526,16 +526,22 @@ App.format = {
}
// HAWQ custom commands on back Ops page.
if (result === ' Resync Hawq Standby HAWQ Standby Master') {
- result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label');
- }
+ result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label');
+ }
if (result === ' Immediate Stop Hawq Service HAWQ Master') {
- result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label');
- }
+ result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label');
+ }
if (result === ' Immediate Stop Hawq Segment HAWQ Segment') {
- result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label');
- }
+ result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label');
+ }
if(result === ' Activate Hawq Standby HAWQ Standby Master') {
- result = Em.I18n.t('admin.activateHawqStandby.button.enable');
+ result = Em.I18n.t('admin.activateHawqStandby.button.enable');
+ }
+ if(result === ' Hawq Clear Cache HAWQ Master') {
+ result = Em.I18n.t('services.service.actions.run.clearHawqCache.label');
+ }
+ if(result === ' Run Hawq Check HAWQ Master') {
+ result = Em.I18n.t('services.service.actions.run.runHawqCheck.label');
}
//<---End HAWQ custom commands--->
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e93caf4/ambari-web/app/views/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js
index 145b64b..7b17e74 100644
--- a/ambari-web/app/views/main/service/item.js
+++ b/ambari-web/app/views/main/service/item.js
@@ -42,10 +42,9 @@ App.MainServiceItemView = Em.View.extend({
'RESOURCEMANAGER': ['DECOMMISSION', 'REFRESHQUEUES'],
'HBASE_MASTER': ['DECOMMISSION'],
'KNOX_GATEWAY': ['STARTDEMOLDAP','STOPDEMOLDAP'],
- 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE'],
+ 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE'],
'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
'HAWQSTANDBY' : ['RESYNC_HAWQ_STANDBY','ACTIVATE_HAWQ_STANDBY']
-
},
addActionMap: function() {