You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/01/03 16:26:16 UTC
git commit: AMBARI-4222. Add historyserver to HDP1 on backend
(aonishuk)
Updated Branches:
refs/heads/trunk bf0aee7f6 -> 13ac36e28
AMBARI-4222. Add historyserver to HDP1 on backend (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/13ac36e2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/13ac36e2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/13ac36e2
Branch: refs/heads/trunk
Commit: 13ac36e28bfec6c40f627001aca421ceca16f627
Parents: bf0aee7
Author: Andrew Onischuk <ao...@hortonworks.com>
Authored: Fri Jan 3 07:24:17 2014 -0800
Committer: Andrew Onischuk <ao...@hortonworks.com>
Committed: Fri Jan 3 07:25:52 2014 -0800
----------------------------------------------------------------------
.../HDFS/package/scripts/hdfs_namenode.py | 11 ++-------
.../services/HDFS/package/scripts/params.py | 2 --
.../MAPREDUCE/configuration/mapred-site.xml | 22 ++++++++++++++++++
.../HDP/1.3.4/services/MAPREDUCE/metainfo.xml | 11 +++++++++
.../MAPREDUCE/package/scripts/historyserver.py | 24 ++++++++++----------
.../MAPREDUCE/package/scripts/jobtracker.py | 9 ++------
.../HDFS/package/scripts/hdfs_namenode.py | 4 ++--
7 files changed, 51 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py
index 1132cd7..9830b63 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/hdfs_namenode.py
@@ -101,23 +101,16 @@ def create_app_directories():
mode="755"
)
if len(params.hs_host) != 0:
- if params.yarn_log_aggregation_enabled:
- hdfs_directory(name=params.yarn_nm_app_log_dir,
- owner=params.yarn_user,
- group=params.user_group,
- mode="1777",
- recursive_chmod=True
- )
hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir,
owner=params.mapred_user,
group=params.user_group,
- mode="1777"
+ mode="777"
)
hdfs_directory(name=params.mapreduce_jobhistory_done_dir,
owner=params.mapred_user,
group=params.user_group,
- mode="1777"
+ mode="777"
)
pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py
index a727aec..e93c50a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/HDFS/package/scripts/params.py
@@ -127,8 +127,6 @@ hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']#",
hbase_staging_dir = "/apps/hbase/staging"
hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.warehouse.dir"] #, "/apps/hive/warehouse")
webhcat_apps_dir = "/apps/webhcat"
-yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']#","true")
-yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']#","/app-logs")
mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir']#","/app-logs")
mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir']#","/mr-history/done")
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml
index da3545c..8367581 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/configuration/mapred-site.xml
@@ -555,6 +555,28 @@
</property>
<property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
+ <description>
+ Directory where history files are written by MapReduce jobs.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
+ <description>
+ Directory where history files are managed by the MR JobHistory Server.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>localhost:19888</value>
+ <description>Enter your JobHistoryServer hostname.</description>
+ </property>
+
+ <property>
<name>mapred.jobtracker.blacklist.fault-timeout-window</name>
<value>180</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml
index a1ca7ba..1e44d2c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/metainfo.xml
@@ -54,6 +54,17 @@
<timeout>600</timeout>
</commandScript>
</component>
+
+
+ <component>
+ <name>HISTORYSERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/historyserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
</components>
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py
index c582a02..8eb2089 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/historyserver.py
@@ -26,14 +26,14 @@ from mapreduce import mapreduce
from service import service
class Historyserver(Script):
- # def install(self, env):
- # self.install_packages(env)
- # self.configure(env)
- #
- # def configure(self, env):
- # import params
- # env.set_params(params)
- # mapreduce()
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ mapreduce()
def start(self, env):
import params
@@ -50,10 +50,10 @@ class Historyserver(Script):
action='stop'
)
- # def status(self, env):
- # import status_params
- # env.set_params(status_params)
- # check_process_status(status_params.historyserver_pid_file)
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.historyserver_pid_file)
if __name__ == "__main__":
Historyserver().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py
index ab33db7..a48cc23 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.4/services/MAPREDUCE/package/scripts/jobtracker.py
@@ -46,7 +46,6 @@ from resource_management import *
from mapreduce import mapreduce
from service import service
-from historyserver import Historyserver
class Jobtracker(Script):
def install(self, env):
@@ -65,9 +64,7 @@ class Jobtracker(Script):
service('jobtracker',
action='start'
)
-
- Historyserver().start(env) #TODO HS will be separate component
-
+
def stop(self, env):
import params
env.set_params(params)
@@ -75,9 +72,7 @@ class Jobtracker(Script):
service('jobtracker',
action='stop'
)
-
- Historyserver().stop(env) #TODO HS will be separate component
-
+
def status(self, env):
import status_params
env.set_params(status_params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/13ac36e2/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
index aafaf72..ecda794 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py
@@ -106,13 +106,13 @@ def create_app_directories():
hdfs_directory(name=params.yarn_nm_app_log_dir,
owner=params.yarn_user,
group=params.user_group,
- mode="1777",
+ mode="777",
recursive_chmod=True
)
hdfs_directory(name=params.mapreduce_jobhistory_intermediate_done_dir,
owner=params.mapred_user,
group=params.user_group,
- mode="1777"
+ mode="777"
)
hdfs_directory(name=params.mapreduce_jobhistory_done_dir,