You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2017/07/01 13:37:20 UTC
[36/50] [abbrv] bigtop git commit: BIGTOP-2821: expose extra config
options for spark
BIGTOP-2821: expose extra config options for spark
Closes #238
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/15242bd3
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/15242bd3
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/15242bd3
Branch: refs/heads/branch-1.2
Commit: 15242bd3618ee8ed4deef82cbefbbabeb554f0bd
Parents: a312ea6
Author: Kevin W Monroe <ke...@canonical.com>
Authored: Fri Jun 23 11:41:54 2017 -0500
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Jul 1 12:52:59 2017 +0000
----------------------------------------------------------------------
bigtop-deploy/puppet/modules/spark/manifests/init.pp | 3 +++
.../puppet/modules/spark/templates/spark-defaults.conf | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/15242bd3/bigtop-deploy/puppet/modules/spark/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index ebb25d0..6fe23dc 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -146,6 +146,9 @@ class spark {
$use_yarn_shuffle_service = false,
$event_log_dir = "hdfs:///var/log/spark/apps",
$history_log_dir = "hdfs:///var/log/spark/apps",
+ $extra_lib_dirs = "/usr/lib/hadoop/lib/native",
+ $driver_mem = "1g",
+ $executor_mem = "1g",
) {
### This is an ungodly hack to deal with the consequence of adding
http://git-wip-us.apache.org/repos/asf/bigtop/blob/15242bd3/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
index a99d9bd..2ef4a04 100644
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
@@ -30,4 +30,9 @@ spark.yarn.historyServer.address <%= @master_host %>:<%= @history_ui_port %>
<% end -%>
spark.history.ui.port <%= @history_ui_port %>
spark.shuffle.service.enabled <%= @use_yarn_shuffle_service %>
-spark.driver.extraJavaOptions -Djava.library.path=/usr/lib/hadoop/lib/native
+<% if @extra_lib_dirs -%>
+spark.driver.extraLibraryPath <%= @extra_lib_dirs %>
+spark.executor.extraLibraryPath <%= @extra_lib_dirs %>
+<% end -%>
+spark.driver.memory <%= @driver_mem %>
+spark.executor.memory <%= @executor_mem %>