bigtop git commit: BIGTOP-2342. Set yarn.log.server.url to point to JH server

[of...@apache.org]

Repository: bigtop
Updated Branches:
  refs/heads/master fe56d273c -> 9fa68d18f


BIGTOP-2342. Set yarn.log.server.url to point to JH server

Signed-off-by: Olaf Flebbe <of...@fleb.be>


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9fa68d18
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9fa68d18
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9fa68d18

Branch: refs/heads/master
Commit: 9fa68d18f97d8519f2db57cfb2e67d3c03e35d27
Parents: fe56d27
Author: Aaron Tokhy <to...@amazon.com>
Authored: Mon Mar 14 19:41:41 2016 -0700
Committer: Olaf Flebbe <of...@fleb.be>
Committed: Sun Apr 17 20:05:19 2016 +0200

----------------------------------------------------------------------
 bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml          | 9 ++++++---
 bigtop-deploy/puppet/modules/hadoop/manifests/init.pp       | 1 +
 bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml | 7 ++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/9fa68d18/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 4943e1c..ac8977c 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -98,6 +98,10 @@ hadoop::common_yarn::hadoop_rm_port: "8032"
 
 hadoop::common_mapred_app::jobtracker_host: "%{hiera('bigtop::hadoop_head_node')}"
 hadoop::common_mapred_app::mapreduce_jobhistory_host: "%{hiera('bigtop::hadoop_head_node')}"
+# actually default but needed for hadoop::common_yarn::yarn_log_server_url here
+bigtop::hadoop_history_server_port: "19888"
+bigtop::hadoop_history_server_url: "http://%{hiera('hadoop::common_mapred_app::mapreduce_jobhistory_host')}:%{hiera('bigtop::hadoop_history_server_port')}"
+hadoop::common_yarn::yarn_log_server_url: "%{hiera('bigtop::hadoop_history_server_url')}/jobhistory/logs"
 
 # actually default but needed for hue::server::webhdfs_url here
 hadoop::httpfs::hadoop_httpfs_port: "14000"
@@ -142,7 +146,6 @@ hadoop_zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 # those are only here because they were present as extlookup keys previously
 bigtop::hadoop_rm_http_port: "8088"
 bigtop::hadoop_rm_proxy_port: "8088"
-bigtop::hadoop_history_server_port: "19888"
 bigtop::sqoop2_server_port: "12000"
 bigtop::hbase_thrift_port: "9090"
 bigtop::hadoop_oozie_port: "11000"
@@ -150,8 +153,8 @@ bigtop::hadoop_oozie_port: "11000"
 hue::server::rm_host: "%{hiera('hadoop::common_yarn::hadoop_rm_host')}"
 hue::server::rm_port: "%{hiera('hadoop::common_yarn::hadoop_rm_port')}"
 hue::server::rm_url: "http://%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_rm_http_port')}"
-hue::server::rm_proxy_url: "http://%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_rm_proxy_port')}"
-hue::server::history_server_url: "http://%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_history_server_port')}"
+hue::server::rm_proxy_url: "http://%{hiera('hadoop::common_yarn::hadoop_ps_host')}:%{hiera('hadoop::common_yarn::hadoop_ps_port')}"
+hue::server::history_server_url: "%{hiera('bigtop::hadoop_history_server_url')}"
 # those use fqdn instead of hadoop_head_node because it's only ever activated
 # on the gatewaynode
 hue::server::webhdfs_url: "http://%{fqdn}:%{hiera('hadoop::httpfs::hadoop_httpfs_port')}/webhdfs/v1"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9fa68d18/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 64b3ee6..9a7176a 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -140,6 +140,7 @@ class hadoop ($hadoop_security_authentication = "simple",
       $hadoop_rm_webapp_port = "8088",
       $hadoop_rt_port = "8025",
       $hadoop_sc_port = "8030",
+      $yarn_log_server_url = undef,
       $yarn_nodemanager_resource_memory_mb = undef,
       $yarn_scheduler_maximum_allocation_mb = undef,
       $yarn_scheduler_minimum_allocation_mb = undef,

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9fa68d18/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
index 6f6c464..054ae65 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
@@ -145,7 +145,12 @@
     <name>yarn.log-aggregation-enable</name>
     <value>true</value>
   </property>
-
+<% if @yarn_log_server_url %>
+  <property>
+    <name>yarn.log.server.url</name>
+    <value><%= @yarn_log_server_url %></value>
+  </property>
+<% end %>
   <property>
     <name>yarn.dispatcher.exit-on-error</name>
     <value>true</value>