You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2015/08/17 07:13:38 UTC
[13/23] ambari git commit: AMBARI-12779: [PluggableStackDefinition]
Remove ambari-server/src/main/resources/stacks/PHD (jluniya)
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log
deleted file mode 100644
index 2010c02..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/balancer.log
+++ /dev/null
@@ -1,29 +0,0 @@
-Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved
-Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB
-Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB
-Jul 28, 2014 5:04:07 PM 2 0 B 5.40 GB 9.79 GB
-Jul 28, 2014 5:05:14 PM 3 0 B 5.06 GB 9.79 GB
-Jul 28, 2014 5:05:50 PM 4 0 B 5.06 GB 9.79 GB
-Jul 28, 2014 5:06:56 PM 5 0 B 4.81 GB 9.79 GB
-Jul 28, 2014 5:07:33 PM 6 0 B 4.80 GB 9.79 GB
-Jul 28, 2014 5:09:11 PM 7 0 B 4.29 GB 9.79 GB
-Jul 28, 2014 5:09:47 PM 8 0 B 4.29 GB 9.79 GB
-Jul 28, 2014 5:11:24 PM 9 0 B 3.89 GB 9.79 GB
-Jul 28, 2014 5:12:00 PM 10 0 B 3.86 GB 9.79 GB
-Jul 28, 2014 5:13:37 PM 11 0 B 3.23 GB 9.79 GB
-Jul 28, 2014 5:15:13 PM 12 0 B 2.53 GB 9.79 GB
-Jul 28, 2014 5:15:49 PM 13 0 B 2.52 GB 9.79 GB
-Jul 28, 2014 5:16:25 PM 14 0 B 2.51 GB 9.79 GB
-Jul 28, 2014 5:17:01 PM 15 0 B 2.39 GB 9.79 GB
-Jul 28, 2014 5:17:37 PM 16 0 B 2.38 GB 9.79 GB
-Jul 28, 2014 5:18:14 PM 17 0 B 2.31 GB 9.79 GB
-Jul 28, 2014 5:18:50 PM 18 0 B 2.30 GB 9.79 GB
-Jul 28, 2014 5:19:26 PM 19 0 B 2.21 GB 9.79 GB
-Jul 28, 2014 5:20:02 PM 20 0 B 2.10 GB 9.79 GB
-Jul 28, 2014 5:20:38 PM 21 0 B 2.06 GB 9.79 GB
-Jul 28, 2014 5:22:14 PM 22 0 B 1.68 GB 9.79 GB
-Jul 28, 2014 5:23:20 PM 23 0 B 1.00 GB 9.79 GB
-Jul 28, 2014 5:23:56 PM 24 0 B 1016.16 MB 9.79 GB
-Jul 28, 2014 5:25:33 PM 25 0 B 30.55 MB 9.79 GB
-The cluster is balanced. Exiting...
-Balancing took 24.858033333333335 minutes
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
deleted file mode 100644
index 0cce48c..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-import time
-import sys
-from threading import Thread
-
-
-def write_function(path, handle, interval):
- with open(path) as f:
- for line in f:
- handle.write(line)
- handle.flush()
- time.sleep(interval)
-
-thread = Thread(target = write_function, args = ('balancer.log', sys.stdout, 1.5))
-thread.start()
-
-threaderr = Thread(target = write_function, args = ('balancer-err.log', sys.stderr, 1.5 * 0.023))
-threaderr.start()
-
-thread.join()
-
-
-def rebalancer_out():
- write_function('balancer.log', sys.stdout)
-
-def rebalancer_err():
- write_function('balancer-err.log', sys.stdout)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py
deleted file mode 100644
index c1a66fb..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/datanode.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from hdfs_datanode import datanode
-from hdfs import hdfs
-
-
-class DataNode(Script):
- def install(self, env):
- import params
-
- self.install_packages(env, params.exclude_packages)
- env.set_params(params)
-
- def start(self, env):
- import params
-
- env.set_params(params)
- self.configure(env)
- datanode(action="start")
-
- def stop(self, env):
- import params
-
- env.set_params(params)
- datanode(action="stop")
-
- def configure(self, env):
- import params
- env.set_params(params)
- hdfs()
- datanode(action="configure")
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
- check_process_status(status_params.datanode_pid_file)
-
-
-if __name__ == "__main__":
- DataNode().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py
deleted file mode 100644
index 873aa15..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs.py
+++ /dev/null
@@ -1,80 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-import sys
-import os
-
-
-def hdfs(name=None):
- import params
-
- # On some OS this folder could be not exists, so we will create it before pushing there files
- Directory(params.limits_conf_dir,
- recursive=True,
- owner='root',
- group='root'
- )
-
- File(os.path.join(params.limits_conf_dir, 'hdfs.conf'),
- owner='root',
- group='root',
- mode=0644,
- content=Template("hdfs.conf.j2")
- )
-
- if params.security_enabled:
- tc_mode = 0644
- tc_owner = "root"
- else:
- tc_mode = None
- tc_owner = params.hdfs_user
-
- if "hadoop-policy" in params.config['configurations']:
- XmlConfig("hadoop-policy.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['hadoop-policy'],
- configuration_attributes=params.config['configuration_attributes']['hadoop-policy'],
- owner=params.hdfs_user,
- group=params.user_group
- )
-
- XmlConfig("hdfs-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['hdfs-site'],
- configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
- owner=params.hdfs_user,
- group=params.user_group
- )
-
- XmlConfig("core-site.xml",
- conf_dir=params.hadoop_conf_dir,
- configurations=params.config['configurations']['core-site'],
- configuration_attributes=params.config['configuration_attributes']['core-site'],
- owner=params.hdfs_user,
- group=params.user_group,
- mode=0644
- )
-
- File(os.path.join(params.hadoop_conf_dir, 'slaves'),
- owner=tc_owner,
- content=Template("slaves.j2")
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py
deleted file mode 100644
index 3b4cf3e..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_client.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from hdfs import hdfs
-from utils import service
-
-
-class HdfsClient(Script):
- def install(self, env):
- import params
-
- self.install_packages(env, params.exclude_packages)
- env.set_params(params)
- self.config(env)
-
- def start(self, env):
- import params
-
- env.set_params(params)
-
- def stop(self, env):
- import params
-
- env.set_params(params)
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
- def config(self, env):
- import params
- hdfs()
- pass
-
-
-if __name__ == "__main__":
- HdfsClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py
deleted file mode 100644
index c93c6e4..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_datanode.py
+++ /dev/null
@@ -1,56 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.dfs_datanode_helper import handle_dfs_data_dir
-from utils import service
-
-
-def create_dirs(data_dir, params):
- """
- :param data_dir: The directory to create
- :param params: parameters
- """
- Directory(data_dir,
- recursive=True,
- mode=0755,
- owner=params.hdfs_user,
- group=params.user_group,
- ignore_failures=True
- )
-
-
-def datanode(action=None):
- import params
- if action == "configure":
- Directory(params.dfs_domain_socket_dir,
- recursive=True,
- mode=0751,
- owner=params.hdfs_user,
- group=params.user_group)
-
- handle_dfs_data_dir(create_dirs, params)
-
- elif action == "start" or action == "stop":
- service(
- action=action, name="datanode",
- user=params.hdfs_user,
- create_pid_dir=True,
- create_log_dir=True
- )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py
deleted file mode 100644
index 31fc2f1..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_namenode.py
+++ /dev/null
@@ -1,160 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from utils import service
-
-
-def namenode(action=None, do_format=True):
- import params
- #we need this directory to be present before any action(HA manual steps for
- #additional namenode)
- if action == "configure":
- create_name_dirs(params.dfs_name_dir)
-
- if action == "start":
- if do_format:
- format_namenode()
- pass
-
- File(params.exclude_file_path,
- content=Template("exclude_hosts_list.j2"),
- owner=params.hdfs_user,
- group=params.user_group
- )
-
- service(
- action="start", name="namenode", user=params.hdfs_user,
- create_pid_dir=True,
- create_log_dir=True
- )
- if params.dfs_ha_enabled:
- dfs_check_nn_status_cmd = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active > /dev/null'")
- else:
- dfs_check_nn_status_cmd = None
-
- namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'")
-
- if params.security_enabled:
- Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
- user = params.hdfs_user)
- Execute(namenode_safe_mode_off,
- tries=40,
- try_sleep=10,
- only_if=dfs_check_nn_status_cmd #skip when HA not active
- )
- create_hdfs_directories(dfs_check_nn_status_cmd)
- if action == "stop":
- service(
- action="stop", name="namenode",
- user=params.hdfs_user
- )
-
- if action == "decommission":
- decommission()
-
-def create_name_dirs(directories):
- import params
-
- dirs = directories.split(",")
- Directory(dirs,
- mode=0755,
- owner=params.hdfs_user,
- group=params.user_group,
- recursive=True
- )
-
-
-def create_hdfs_directories(check):
- import params
-
- params.HdfsDirectory("/tmp",
- action="create_delayed",
- owner=params.hdfs_user,
- mode=0777
- )
- params.HdfsDirectory(params.smoke_hdfs_user_dir,
- action="create_delayed",
- owner=params.smoke_user,
- mode=params.smoke_hdfs_user_mode
- )
- params.HdfsDirectory(None, action="create",
- only_if=check #skip creation when HA not active
- )
-
-def format_namenode(force=None):
- import params
-
- old_mark_dir = params.namenode_formatted_old_mark_dir
- mark_dir = params.namenode_formatted_mark_dir
- dfs_name_dir = params.dfs_name_dir
- hdfs_user = params.hdfs_user
- hadoop_conf_dir = params.hadoop_conf_dir
-
- if not params.dfs_ha_enabled:
- if force:
- ExecuteHadoop('namenode -format',
- kinit_override=True,
- bin_dir=params.hadoop_bin_dir,
- conf_dir=hadoop_conf_dir)
- else:
- File(format("{tmp_dir}/checkForFormat.sh"),
- content=StaticFile("checkForFormat.sh"),
- mode=0755)
- Execute(format(
- "{tmp_dir}/checkForFormat.sh {hdfs_user} {hadoop_conf_dir} "
- "{hadoop_bin_dir} {old_mark_dir} {mark_dir} {dfs_name_dir}"),
- not_if=format("test -d {old_mark_dir} || test -d {mark_dir}"),
- path="/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin"
- )
-
- Directory(mark_dir,
- recursive = True
- )
-
-
-def decommission():
- import params
-
- hdfs_user = params.hdfs_user
- conf_dir = params.hadoop_conf_dir
- user_group = params.user_group
- dn_kinit_cmd = params.dn_kinit_cmd
-
- File(params.exclude_file_path,
- content=Template("exclude_hosts_list.j2"),
- owner=hdfs_user,
- group=user_group
- )
-
- Execute(dn_kinit_cmd,
- user=hdfs_user
- )
-
- if params.dfs_ha_enabled:
- # due to a bug in hdfs, refreshNodes will not run on both namenodes so we
- # need to execute each command scoped to a particular namenode
- nn_refresh_cmd = format('dfsadmin -fs hdfs://{namenode_rpc} -refreshNodes')
- else:
- nn_refresh_cmd = format('dfsadmin -refreshNodes')
- ExecuteHadoop(nn_refresh_cmd,
- user=hdfs_user,
- conf_dir=conf_dir,
- kinit_override=True,
- bin_dir=params.hadoop_bin_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py
deleted file mode 100644
index 1dc545e..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_rebalance.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import re
-
-class HdfsParser():
- def __init__(self):
- self.initialLine = None
- self.state = None
-
- def parseLine(self, line):
- hdfsLine = HdfsLine()
- type, matcher = hdfsLine.recognizeType(line)
- if(type == HdfsLine.LineType.HeaderStart):
- self.state = 'PROCESS_STARTED'
- elif (type == HdfsLine.LineType.Progress):
- self.state = 'PROGRESS'
- hdfsLine.parseProgressLog(line, matcher)
- if(self.initialLine == None): self.initialLine = hdfsLine
-
- return hdfsLine
- elif (type == HdfsLine.LineType.ProgressEnd):
- self.state = 'PROCESS_FINISED'
- return None
-
-class HdfsLine():
-
- class LineType:
- HeaderStart, Progress, ProgressEnd, Unknown = range(4)
-
-
- MEMORY_SUFFIX = ['B','KB','MB','GB','TB','PB','EB']
- MEMORY_PATTERN = '(?P<memmult_%d>(?P<memory_%d>(\d+)(.|,)?(\d+)?) (?P<mult_%d>'+"|".join(MEMORY_SUFFIX)+'))'
-
- HEADER_BEGIN_PATTERN = re.compile('Time Stamp\w+Iteration#\w+Bytes Already Moved\w+Bytes Left To Move\w+Bytes Being Moved')
- PROGRESS_PATTERN = re.compile(
- "(?P<date>.*?)\s+" +
- "(?P<iteration>\d+)\s+" +
- MEMORY_PATTERN % (1,1,1) + "\s+" +
- MEMORY_PATTERN % (2,2,2) + "\s+" +
- MEMORY_PATTERN % (3,3,3)
- )
- PROGRESS_END_PATTERN = re.compile('(The cluster is balanced. Exiting...|The cluster is balanced. Exiting...)')
-
- def __init__(self):
- self.date = None
- self.iteration = None
- self.bytesAlreadyMoved = None
- self.bytesLeftToMove = None
- self.bytesBeingMoved = None
- self.bytesAlreadyMovedStr = None
- self.bytesLeftToMoveStr = None
- self.bytesBeingMovedStr = None
-
- def recognizeType(self, line):
- for (type, pattern) in (
- (HdfsLine.LineType.HeaderStart, self.HEADER_BEGIN_PATTERN),
- (HdfsLine.LineType.Progress, self.PROGRESS_PATTERN),
- (HdfsLine.LineType.ProgressEnd, self.PROGRESS_END_PATTERN)
- ):
- m = re.match(pattern, line)
- if m:
- return type, m
- return HdfsLine.LineType.Unknown, None
-
- def parseProgressLog(self, line, m):
- '''
- Parse the line of 'hdfs rebalancer' output. The example output being parsed:
-
- Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved
- Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB
- Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB
-
- Throws AmbariException in case of parsing errors
-
- '''
- m = re.match(self.PROGRESS_PATTERN, line)
- if m:
- self.date = m.group('date')
- self.iteration = int(m.group('iteration'))
-
- self.bytesAlreadyMoved = self.parseMemory(m.group('memory_1'), m.group('mult_1'))
- self.bytesLeftToMove = self.parseMemory(m.group('memory_2'), m.group('mult_2'))
- self.bytesBeingMoved = self.parseMemory(m.group('memory_3'), m.group('mult_3'))
-
- self.bytesAlreadyMovedStr = m.group('memmult_1')
- self.bytesLeftToMoveStr = m.group('memmult_2')
- self.bytesBeingMovedStr = m.group('memmult_3')
- else:
- raise AmbariException("Failed to parse line [%s]")
-
- def parseMemory(self, memorySize, multiplier_type):
- try:
- factor = self.MEMORY_SUFFIX.index(multiplier_type)
- except ValueError:
- raise AmbariException("Failed to memory value [%s %s]" % (memorySize, multiplier_type))
-
- return float(memorySize) * (1024 ** factor)
- def toJson(self):
- return {
- 'timeStamp' : self.date,
- 'iteration' : self.iteration,
-
- 'dataMoved': self.bytesAlreadyMovedStr,
- 'dataLeft' : self.bytesLeftToMoveStr,
- 'dataBeingMoved': self.bytesBeingMovedStr,
-
- 'bytesMoved': self.bytesAlreadyMoved,
- 'bytesLeft' : self.bytesLeftToMove,
- 'bytesBeingMoved': self.bytesBeingMoved,
- }
- def __str__(self):
- return "[ date=%s,iteration=%d, bytesAlreadyMoved=%d, bytesLeftToMove=%d, bytesBeingMoved=%d]"%(self.date, self.iteration, self.bytesAlreadyMoved, self.bytesLeftToMove, self.bytesBeingMoved)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py
deleted file mode 100644
index 49241b4..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/hdfs_snamenode.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from utils import service
-from utils import hdfs_directory
-
-
-def snamenode(action=None, format=False):
- import params
-
- if action == "configure":
- Directory(params.fs_checkpoint_dir,
- recursive=True,
- mode=0755,
- owner=params.hdfs_user,
- group=params.user_group)
- File(params.exclude_file_path,
- content=Template("exclude_hosts_list.j2"),
- owner=params.hdfs_user,
- group=params.user_group)
- elif action == "start" or action == "stop":
- service(
- action=action,
- name="secondarynamenode",
- user=params.hdfs_user,
- create_pid_dir=True,
- create_log_dir=True
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py
deleted file mode 100644
index de18c88..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/journalnode.py
+++ /dev/null
@@ -1,73 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from utils import service
-from hdfs import hdfs
-
-
-class JournalNode(Script):
- def install(self, env):
- import params
-
- self.install_packages(env, params.exclude_packages)
- env.set_params(params)
-
- def start(self, env):
- import params
-
- env.set_params(params)
- self.configure(env)
- service(
- action="start", name="journalnode", user=params.hdfs_user,
- create_pid_dir=True,
- create_log_dir=True
- )
-
- def stop(self, env):
- import params
-
- env.set_params(params)
- service(
- action="stop", name="journalnode", user=params.hdfs_user,
- create_pid_dir=True,
- create_log_dir=True
- )
-
- def configure(self, env):
- import params
-
- Directory(params.jn_edits_dir,
- recursive=True,
- owner=params.hdfs_user,
- group=params.user_group
- )
- env.set_params(params)
- hdfs()
- pass
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
- check_process_status(status_params.journalnode_pid_file)
-
-
-if __name__ == "__main__":
- JournalNode().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py
deleted file mode 100644
index a0b07aa..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/namenode.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from hdfs_namenode import namenode
-from hdfs import hdfs
-import time
-import json
-import subprocess
-import hdfs_rebalance
-import sys
-import os
-from datetime import datetime
-
-
-class NameNode(Script):
- def install(self, env):
- import params
-
- self.install_packages(env, params.exclude_packages)
- env.set_params(params)
- #TODO we need this for HA because of manual steps
- self.configure(env)
-
- def start(self, env):
- import params
-
- env.set_params(params)
- self.configure(env)
- namenode(action="start")
-
- def stop(self, env):
- import params
-
- env.set_params(params)
- namenode(action="stop")
-
- def configure(self, env):
- import params
-
- env.set_params(params)
- hdfs()
- namenode(action="configure")
- pass
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
- Execute(format("echo '{namenode_pid_file}' >> /1.txt"))
- check_process_status(status_params.namenode_pid_file)
- pass
-
- def decommission(self, env):
- import params
-
- env.set_params(params)
- namenode(action="decommission")
- pass
-
-
- def rebalancehdfs(self, env):
- import params
- env.set_params(params)
-
- name_node_parameters = json.loads( params.name_node_params )
- threshold = name_node_parameters['threshold']
- _print("Starting balancer with threshold = %s\n" % threshold)
-
- def calculateCompletePercent(first, current):
- return 1.0 - current.bytesLeftToMove/first.bytesLeftToMove
-
-
- def startRebalancingProcess(threshold):
- rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} balancer -threshold {threshold}')
- return ['su','-',params.hdfs_user,'-c', rebalanceCommand]
-
- command = startRebalancingProcess(threshold)
-
- basedir = os.path.join(env.config.basedir, 'scripts')
- if(threshold == 'DEBUG'): #FIXME TODO remove this on PROD
- basedir = os.path.join(env.config.basedir, 'scripts', 'balancer-emulator')
- command = ['python','hdfs-command.py']
-
- _print("Executing command %s\n" % command)
-
- parser = hdfs_rebalance.HdfsParser()
- proc = subprocess.Popen(
- command,
- stdout=subprocess.PIPE,
- shell=False,
- close_fds=True,
- cwd=basedir
- )
- for line in iter(proc.stdout.readline, ''):
- _print('[balancer] %s %s' % (str(datetime.now()), line ))
- pl = parser.parseLine(line)
- if pl:
- res = pl.toJson()
- res['completePercent'] = calculateCompletePercent(parser.initialLine, pl)
-
- self.put_structured_out(res)
- elif parser.state == 'PROCESS_FINISED' :
- _print('[balancer] %s %s' % (str(datetime.now()), 'Process is finished' ))
- self.put_structured_out({'completePercent' : 1})
- break
-
- proc.stdout.close()
- proc.wait()
- if proc.returncode != None and proc.returncode != 0:
- raise Fail('Hdfs rebalance process exited with error. See the log output')
-
-def _print(line):
- sys.stdout.write(line)
- sys.stdout.flush()
-
-if __name__ == "__main__":
- NameNode().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py
deleted file mode 100644
index 8c2dacb..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/params.py
+++ /dev/null
@@ -1,235 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import status_params
-import os
-
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-#RPM versioning support
-rpm_version = default("/configurations/cluster-env/rpm_version", None)
-
-#hadoop params
-if rpm_version:
- mapreduce_libs_path = "/usr/phd/current/hadoop-mapreduce-client/*"
- hadoop_libexec_dir = "/usr/phd/current/hadoop-client/libexec"
- hadoop_bin = "/usr/phd/current/hadoop-client/sbin"
- hadoop_bin_dir = "/usr/phd/current/hadoop-client/bin"
-else:
- mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
- hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
- hadoop_bin = "/usr/lib/hadoop/sbin"
- hadoop_bin_dir = "/usr/bin"
-
-hadoop_conf_dir = "/etc/hadoop/conf"
-hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
-limits_conf_dir = "/etc/security/limits.d"
-
-execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir
-ulimit_cmd = "ulimit -c unlimited; "
-
-#security params
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-falcon_user = config['configurations']['falcon-env']['falcon_user']
-
-#exclude file
-hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", [])
-exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude']
-update_exclude_file_only = config['commandParams']['update_exclude_file_only']
-
-kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-#hosts
-hostname = config["hostname"]
-rm_host = default("/clusterHostInfo/rm_host", [])
-slave_hosts = default("/clusterHostInfo/slave_hosts", [])
-hagios_server_hosts = default("/clusterHostInfo/nagios_server_host", [])
-oozie_servers = default("/clusterHostInfo/oozie_server", [])
-hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
-hive_server_host = default("/clusterHostInfo/hive_server_host", [])
-hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
-hs_host = default("/clusterHostInfo/hs_host", [])
-jtnode_host = default("/clusterHostInfo/jtnode_host", [])
-namenode_host = default("/clusterHostInfo/namenode_host", [])
-nm_host = default("/clusterHostInfo/nm_host", [])
-ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
-journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", [])
-zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", [])
-falcon_host = default("/clusterHostInfo/falcon_server_hosts", [])
-
-has_ganglia_server = not len(ganglia_server_hosts) == 0
-has_namenodes = not len(namenode_host) == 0
-has_jobtracker = not len(jtnode_host) == 0
-has_resourcemanager = not len(rm_host) == 0
-has_histroryserver = not len(hs_host) == 0
-has_hbase_masters = not len(hbase_master_hosts) == 0
-has_slaves = not len(slave_hosts) == 0
-has_nagios = not len(hagios_server_hosts) == 0
-has_oozie_server = not len(oozie_servers) == 0
-has_hcat_server_host = not len(hcat_server_hosts) == 0
-has_hive_server_host = not len(hive_server_host) == 0
-has_journalnode_hosts = not len(journalnode_hosts) == 0
-has_zkfc_hosts = not len(zkfc_hosts) == 0
-has_falcon_host = not len(falcon_host) == 0
-
-
-is_namenode_master = hostname in namenode_host
-is_jtnode_master = hostname in jtnode_host
-is_rmnode_master = hostname in rm_host
-is_hsnode_master = hostname in hs_host
-is_hbase_master = hostname in hbase_master_hosts
-is_slave = hostname in slave_hosts
-
-if has_ganglia_server:
- ganglia_server_host = ganglia_server_hosts[0]
-
-#users and groups
-yarn_user = config['configurations']['yarn-env']['yarn_user']
-hbase_user = config['configurations']['hbase-env']['hbase_user']
-nagios_user = config['configurations']['nagios-env']['nagios_user']
-oozie_user = config['configurations']['oozie-env']['oozie_user']
-webhcat_user = config['configurations']['hive-env']['hcat_user']
-hcat_user = config['configurations']['hive-env']['hcat_user']
-hive_user = config['configurations']['hive-env']['hive_user']
-smoke_user = config['configurations']['cluster-env']['smokeuser']
-mapred_user = config['configurations']['mapred-env']['mapred_user']
-hdfs_user = status_params.hdfs_user
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-
-user_group = config['configurations']['cluster-env']['user_group']
-proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group']
-nagios_group = config['configurations']['nagios-env']['nagios_group']
-
-#hadoop params
-hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix
-
-hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
-hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
-
-dfs_domain_socket_path = config['configurations']['hdfs-site']['dfs.domain.socket.path']
-dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path)
-
-jn_edits_dir = config['configurations']['hdfs-site']['dfs.journalnode.edits.dir']
-
-dfs_name_dir = config['configurations']['hdfs-site']['dfs.namenode.name.dir']
-
-namenode_dirs_created_stub_dir = format("{hdfs_log_dir_prefix}/{hdfs_user}")
-namenode_dirs_stub_filename = "namenode_dirs_created"
-
-smoke_hdfs_user_dir = format("/user/{smoke_user}")
-smoke_hdfs_user_mode = 0770
-
-namenode_formatted_old_mark_dir = format("{hadoop_pid_dir_prefix}/hdfs/namenode/formatted/")
-namenode_formatted_mark_dir = format("/var/lib/hdfs/namenode/formatted/")
-
-fs_checkpoint_dir = config['configurations']['hdfs-site']['dfs.namenode.checkpoint.dir']
-
-dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir']
-data_dir_mount_file = config['configurations']['hadoop-env']['dfs.datanode.data.dir.mount.file']
-
-dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None)
-dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None)
-dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None)
-dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None)
-
-# HDFS High Availability properties
-dfs_ha_enabled = False
-dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.nameservices", None)
-dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
-
-namenode_id = None
-namenode_rpc = None
-
-if dfs_ha_namenode_ids:
- dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
- dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
- if dfs_ha_namenode_ids_array_len > 1:
- dfs_ha_enabled = True
-if dfs_ha_enabled:
- for nn_id in dfs_ha_namemodes_ids_list:
- nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')]
- if hostname in nn_host:
- namenode_id = nn_id
- namenode_rpc = nn_host
-
-journalnode_address = default('/configurations/hdfs-site/dfs.journalnode.http-address', None)
-if journalnode_address:
- journalnode_port = journalnode_address.split(":")[1]
-
-
-if security_enabled:
- _dn_principal_name = config['configurations']['hdfs-site']['dfs.datanode.kerberos.principal']
- _dn_keytab = config['configurations']['hdfs-site']['dfs.datanode.keytab.file']
- _dn_principal_name = _dn_principal_name.replace('_HOST',hostname.lower())
-
- dn_kinit_cmd = format("{kinit_path_local} -kt {_dn_keytab} {_dn_principal_name};")
-else:
- dn_kinit_cmd = ""
-
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- bin_dir = hadoop_bin_dir
-)
-
-io_compression_codecs = config['configurations']['core-site']['io.compression.codecs']
-if not "com.hadoop.compression.lzo" in io_compression_codecs:
- exclude_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native", "liblzo2-2"]
-else:
- exclude_packages = []
-name_node_params = default("/commandParams/namenode", None)
-
-#hadoop params
-hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
-
-#hadoop-env.sh
-java_home = config['hostLevelParams']['java_home']
-stack_version = str(config['hostLevelParams']['stack_version'])
-
-stack_is_champlain_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1'))
-
-if stack_version.startswith('2.0') and System.get_instance().os_family != "suse":
- # deprecated rhel jsvc_path
- jsvc_path = "/usr/libexec/phd-utils"
-else:
- jsvc_path = "/usr/lib/phd-utils"
-
-hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
-namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
-namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
-namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-
-jtnode_opt_newsize = "200m"
-jtnode_opt_maxnewsize = "200m"
-jtnode_heapsize = "1024m"
-ttnode_heapsize = "1024m"
-
-dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
-mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
-mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py
deleted file mode 100644
index 36e26d6..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/service_check.py
+++ /dev/null
@@ -1,120 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-
-class HdfsServiceCheck(Script):
- def service_check(self, env):
- import params
-
- env.set_params(params)
- unique = functions.get_unique_id_and_date()
- dir = '/tmp'
- tmp_file = format("{dir}/{unique}")
-
- safemode_command = "dfsadmin -safemode get | grep OFF"
-
- create_dir_cmd = format("fs -mkdir {dir}")
- chmod_command = format("fs -chmod 777 {dir}")
- test_dir_exists = format("su -s /bin/bash - {smoke_user} -c '{hadoop_bin_dir}/hadoop --config {hadoop_conf_dir} fs -test -e {dir}'")
- cleanup_cmd = format("fs -rm {tmp_file}")
- #cleanup put below to handle retries; if retrying there wil be a stale file
- #that needs cleanup; exit code is fn of second command
- create_file_cmd = format(
- "{cleanup_cmd}; hadoop --config {hadoop_conf_dir} fs -put /etc/passwd {tmp_file}")
- test_cmd = format("fs -test -e {tmp_file}")
- if params.security_enabled:
- Execute(format(
- "su -s /bin/bash - {smoke_user} -c '{kinit_path_local} -kt {smoke_user_keytab} "
- "{smoke_user}'"))
- ExecuteHadoop(safemode_command,
- user=params.smoke_user,
- logoutput=True,
- conf_dir=params.hadoop_conf_dir,
- try_sleep=3,
- tries=20,
- bin_dir=params.hadoop_bin_dir
- )
- ExecuteHadoop(create_dir_cmd,
- user=params.smoke_user,
- logoutput=True,
- not_if=test_dir_exists,
- conf_dir=params.hadoop_conf_dir,
- try_sleep=3,
- tries=5,
- bin_dir=params.hadoop_bin_dir
- )
- ExecuteHadoop(chmod_command,
- user=params.smoke_user,
- logoutput=True,
- conf_dir=params.hadoop_conf_dir,
- try_sleep=3,
- tries=5,
- bin_dir=params.hadoop_bin_dir
- )
- ExecuteHadoop(create_file_cmd,
- user=params.smoke_user,
- logoutput=True,
- conf_dir=params.hadoop_conf_dir,
- try_sleep=3,
- tries=5,
- bin_dir=params.hadoop_bin_dir
- )
- ExecuteHadoop(test_cmd,
- user=params.smoke_user,
- logoutput=True,
- conf_dir=params.hadoop_conf_dir,
- try_sleep=3,
- tries=5,
- bin_dir=params.hadoop_bin_dir
- )
- if params.has_journalnode_hosts:
- journalnode_port = params.journalnode_port
- smoke_test_user = params.smoke_user
- checkWebUIFileName = "checkWebUI.py"
- checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
- comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
- checkWebUICmd = format(
- "su -s /bin/bash - {smoke_test_user} -c 'python {checkWebUIFilePath} -m "
- "{comma_sep_jn_hosts} -p {journalnode_port}'")
- File(checkWebUIFilePath,
- content=StaticFile(checkWebUIFileName))
-
- Execute(checkWebUICmd,
- logoutput=True,
- try_sleep=3,
- tries=5
- )
-
- if params.is_namenode_master:
- if params.has_zkfc_hosts:
- pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
- pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
- check_zkfc_process_cmd = format(
- "ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
- Execute(check_zkfc_process_cmd,
- logoutput=True,
- try_sleep=3,
- tries=5
- )
-
-
-if __name__ == "__main__":
- HdfsServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py
deleted file mode 100644
index 5eb25d2..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/snamenode.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from hdfs_snamenode import snamenode
-from hdfs import hdfs
-
-
-class SNameNode(Script):
- def install(self, env):
- import params
-
- env.set_params(params)
-
- self.install_packages(env, params.exclude_packages)
-
-
- def start(self, env):
- import params
-
- env.set_params(params)
-
- self.configure(env)
- snamenode(action="start")
-
- def stop(self, env):
- import params
-
- env.set_params(params)
-
- snamenode(action="stop")
-
- def configure(self, env):
- import params
-
- env.set_params(params)
- hdfs()
- snamenode(action="configure")
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
-
- check_process_status(status_params.snamenode_pid_file)
-
-
-if __name__ == "__main__":
- SNameNode().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py
deleted file mode 100644
index 1dd4750..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/status_params.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-config = Script.get_config()
-
-hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-phd_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
-datanode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-datanode.pid")
-namenode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-namenode.pid")
-snamenode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-secondarynamenode.pid")
-journalnode_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-journalnode.pid")
-zkfc_pid_file = format("{phd_pid_dir}/hadoop-{hdfs_user}-zkfc.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py
deleted file mode 100644
index 5e1221a..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/utils.py
+++ /dev/null
@@ -1,149 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import os
-
-from resource_management import *
-import re
-
-
-def service(action=None, name=None, user=None, create_pid_dir=False,
- create_log_dir=False):
- import params
-
- pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
- pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
- log_dir = format("{hdfs_log_dir_prefix}/{user}")
- check_process = format(
- "ls {pid_file} >/dev/null 2>&1 &&"
- " ps `cat {pid_file}` >/dev/null 2>&1")
-
- if create_pid_dir:
- Directory(pid_dir,
- owner=user,
- recursive=True)
- if create_log_dir:
- Directory(log_dir,
- owner=user,
- recursive=True)
-
- hadoop_env_exports = {
- 'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir
- }
-
- if params.security_enabled and name == "datanode":
- dfs_dn_port = get_port(params.dfs_dn_addr)
- dfs_dn_http_port = get_port(params.dfs_dn_http_addr)
- dfs_dn_https_port = get_port(params.dfs_dn_https_addr)
-
- # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
- if params.dfs_http_policy == "HTTPS_ONLY":
- secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port)
- elif params.dfs_http_policy == "HTTP_AND_HTTPS":
- secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port)
- else: # params.dfs_http_policy == "HTTP_ONLY" or not defined:
- secure_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port)
-
- # Calculate HADOOP_SECURE_DN_* env vars, but not append them yet
- # These variables should not be set when starting secure datanode as a non-root
- ## On secure datanodes, user to run the datanode as after dropping privileges
- hadoop_secure_dn_user = params.hdfs_user
- ## Where log files are stored in the secure data environment.
- hadoop_secure_dn_log_dir = format("{hdfs_log_dir_prefix}/{hadoop_secure_dn_user}")
- ## The directory where pid files are stored in the secure data environment.
- hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hadoop_secure_dn_user}")
- hadoop_secure_dn_exports = {
- 'HADOOP_SECURE_DN_USER' : hadoop_secure_dn_user,
- 'HADOOP_SECURE_DN_LOG_DIR' : hadoop_secure_dn_log_dir,
- 'HADOOP_SECURE_DN_PID_DIR' : hadoop_secure_dn_pid_dir
- }
- hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
-
- # At Champlain stack and further, we may start datanode as a non-root even in secure cluster
- if not params.stack_is_champlain_or_further or secure_ports_are_in_use:
- user = "root"
- pid_file = format(
- "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid")
- if params.stack_is_champlain_or_further:
- hadoop_env_exports.update(hadoop_secure_dn_exports)
-
- if action == 'stop' and params.stack_is_champlain_or_further and \
- os.path.isfile(hadoop_secure_dn_pid_file):
- # We need special handling for this case to handle the situation
- # when we configure non-root secure DN and then restart it
- # to handle new configs. Otherwise we will not be able to stop
- # a running instance
- user = "root"
- try:
- with open(hadoop_secure_dn_pid_file, 'r') as f:
- pid = f.read()
- os.kill(int(pid), 0)
- hadoop_env_exports.update(hadoop_secure_dn_exports)
- except IOError:
- pass # Can not open pid file
- except ValueError:
- pass # Pid file content is invalid
- except OSError:
- pass # Process is not running
-
-
- hadoop_env_exports_str = ''
- for exp in hadoop_env_exports.items():
- hadoop_env_exports_str += "export {0}={1} && ".format(exp[0], exp[1])
-
- hadoop_daemon = format(
- "{hadoop_env_exports_str}"
- "{hadoop_bin}/hadoop-daemon.sh")
- cmd = format("{hadoop_daemon} --config {hadoop_conf_dir}")
-
- daemon_cmd = format("{ulimit_cmd} su -s /bin/bash - {user} -c '{cmd} {action} {name}'")
-
- service_is_up = check_process if action == "start" else None
- #remove pid file from dead process
- File(pid_file,
- action="delete",
- not_if=check_process,
- )
- Execute(daemon_cmd,
- not_if=service_is_up
- )
- if action == "stop":
- File(pid_file,
- action="delete",
- )
-
-def get_port(address):
- """
- Extracts port from the address like 0.0.0.0:1019
- """
- if address is None:
- return None
- m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address)
- if m is not None:
- return int(m.group(2))
- else:
- return None
-
-def is_secure_port(port):
- """
- Returns True if port is root-owned at *nix systems
- """
- if port is not None:
- return port < 1024
- else:
- return False
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py
deleted file mode 100644
index fd9bbfa..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/scripts/zkfc_slave.py
+++ /dev/null
@@ -1,64 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-from utils import service
-from hdfs import hdfs
-
-
-class ZkfcSlave(Script):
- def install(self, env):
- import params
-
- self.install_packages(env, params.exclude_packages)
- env.set_params(params)
-
- def start(self, env):
- import params
-
- env.set_params(params)
- self.configure(env)
- service(
- action="start", name="zkfc", user=params.hdfs_user, create_pid_dir=True,
- create_log_dir=True
- )
-
- def stop(self, env):
- import params
-
- env.set_params(params)
- service(
- action="stop", name="zkfc", user=params.hdfs_user, create_pid_dir=True,
- create_log_dir=True
- )
-
- def configure(self, env):
- hdfs()
- pass
-
- def status(self, env):
- import status_params
-
- env.set_params(status_params)
-
- check_process_status(status_params.zkfc_pid_file)
-
-
-if __name__ == "__main__":
- ZkfcSlave().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2
deleted file mode 100644
index a92cdc1..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/exclude_hosts_list.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-{% for host in hdfs_exclude_file %}
-{{host}}
-{% endfor %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2
deleted file mode 100644
index d58a6f5..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/hdfs.conf.j2
+++ /dev/null
@@ -1,35 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{hdfs_user}} - nofile 32768
-{{hdfs_user}} - nproc 65536
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2 b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2
deleted file mode 100644
index 4a9e713..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HDFS/package/templates/slaves.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-{% for host in slave_hosts %}
-{{host}}
-{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml
deleted file mode 100644
index 91b402b..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hcat-env.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <!-- hcat-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for hcat-env.sh file</description>
- <value>
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
-
- JAVA_HOME={{java64_home}}
- HCAT_PID_DIR={{hcat_pid_dir}}/
- HCAT_LOG_DIR={{hcat_log_dir}}/
- HCAT_CONF_DIR={{hcat_conf_dir}}
- HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
- #DBROOT is the path where the connector jars are downloaded
- DBROOT={{hcat_dbroot}}
- USER={{hcat_user}}
- METASTORE_PORT={{hive_metastore_port}}
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml
deleted file mode 100644
index 1b8b75a..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-env.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>hive_database_type</name>
- <value>postgres</value>
- <description>Default HIVE DB type.</description>
- </property>
- <property>
- <name>hive_database</name>
- <value>New PostgreSQL Database</value>
- <description>
- Property that determines whether the HIVE DB is managed by Ambari.
- </description>
- </property>
- <property>
- <name>hive_ambari_database</name>
- <value>PostgreSQL</value>
- <description>Database type.</description>
- </property>
- <property>
- <name>hive_database_name</name>
- <value>hive</value>
- <description>Database name.</description>
- </property>
- <property>
- <name>hive_dbroot</name>
- <value>/usr/lib/hive/lib/</value>
- <description>Hive DB Directory.</description>
- </property>
- <property>
- <name>hive_log_dir</name>
- <value>/var/log/hive</value>
- <description>Directory for Hive Log files.</description>
- </property>
- <property>
- <name>hive_pid_dir</name>
- <value>/var/run/hive</value>
- <description>Hive PID Dir.</description>
- </property>
- <property>
- <name>hive_user</name>
- <value>hive</value>
- <property-type>USER</property-type>
- <description>Hive User.</description>
- </property>
-
- <!--HCAT-->
-
- <property>
- <name>hcat_log_dir</name>
- <value>/var/log/webhcat</value>
- <description>WebHCat Log Dir.</description>
- </property>
- <property>
- <name>hcat_pid_dir</name>
- <value>/var/run/webhcat</value>
- <description>WebHCat Pid Dir.</description>
- </property>
- <property>
- <name>hcat_user</name>
- <value>hcat</value>
- <property-type>USER</property-type>
- <description>HCat User.</description>
- </property>
- <property>
- <name>webhcat_user</name>
- <value>hcat</value>
- <property-type>USER</property-type>
- <description>WebHCat User.</description>
- </property>
-
- <!-- hive-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for hive-env.sh file</description>
- <value>
- if [ "$SERVICE" = "cli" ]; then
- if [ -z "$DEBUG" ]; then
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit"
- else
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
- fi
- fi
-
-# The heap size of the jvm stared by hive shell script can be controlled via:
-
-export HADOOP_HEAPSIZE="{{hive_heapsize}}"
-export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS"
-
-# Larger heap size may be required when running queries over large number of files or partitions.
-# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
-# appropriate for hive server (hwi etc).
-
-
-# Set HADOOP_HOME to point to a specific hadoop install directory
-HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
-
-# Hive Configuration Directory can be controlled by:
-export HIVE_CONF_DIR={{hive_config_dir}}
-
-# Folder containing extra ibraries required for hive compilation/execution can be controlled by:
-if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
- export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
-elif [ -d "/usr/lib/hive-hcatalog/" ]; then
- export HIVE_AUX_JARS_PATH=/usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core-*.jar
-else
- export HIVE_AUX_JARS_PATH=/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar
-fi
-export METASTORE_PORT={{hive_metastore_port}}
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml
deleted file mode 100644
index fb852f7..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-exec-log4j.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Custom hive-exec-log4j</description>
- <value>
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-
-hive.log.threshold=ALL
-hive.root.logger=INFO,FA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.query.id=hadoop
-hive.log.file=${hive.query.id}.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=${hive.log.threshold}
-
-#
-# File Appender
-#
-
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,FA
-log4j.category.Datastore=ERROR,FA
-log4j.category.Datastore.Schema=ERROR,FA
-log4j.category.JPOX.Datastore=ERROR,FA
-log4j.category.JPOX.Plugin=ERROR,FA
-log4j.category.JPOX.MetaData=ERROR,FA
-log4j.category.JPOX.Query=ERROR,FA
-log4j.category.JPOX.General=ERROR,FA
-log4j.category.JPOX.Enhancer=ERROR,FA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
-
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/930d4499/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml b/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml
deleted file mode 100644
index cb8d3a9..0000000
--- a/ambari-server/src/main/resources/stacks/PHD/3.0.0.0/services/HIVE/configuration/hive-log4j.xml
+++ /dev/null
@@ -1,120 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Custom log4j.properties</description>
- <value>
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=INFO,DRFA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.log.file=hive.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=${hive.log.threshold}
-
-#
-# Daily Rolling File Appender
-#
-# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
-# for different CLI session.
-#
-# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-
-log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.console.encoding=UTF-8
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,DRFA
-log4j.category.Datastore=ERROR,DRFA
-log4j.category.Datastore.Schema=ERROR,DRFA
-log4j.category.JPOX.Datastore=ERROR,DRFA
-log4j.category.JPOX.Plugin=ERROR,DRFA
-log4j.category.JPOX.MetaData=ERROR,DRFA
-log4j.category.JPOX.Query=ERROR,DRFA
-log4j.category.JPOX.General=ERROR,DRFA
-log4j.category.JPOX.Enhancer=ERROR,DRFA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
- </value>
- </property>
-
-</configuration>