[bigtop] branch master updated: BIGTOP-3767. Add Spark Thrift Server Puppet Deploy Code (#973)

[iw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new 0aa921cd BIGTOP-3767. Add Spark Thrift Server Puppet Deploy Code (#973)
0aa921cd is described below

commit 0aa921cd49a97e90393c27203c7a907f3ab7f87d
Author: Peng Lee <49...@users.noreply.github.com>
AuthorDate: Sun Aug 14 14:05:03 2022 +0800

    BIGTOP-3767. Add Spark Thrift Server Puppet Deploy Code (#973)
---
 bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml |  2 ++
 bigtop-deploy/puppet/manifests/cluster.pp          |  1 +
 .../puppet/modules/spark/manifests/init.pp         | 27 ++++++++++++++++++++++
 .../modules/spark/templates/spark-defaults.conf    |  8 ++++++-
 .../puppet/modules/spark/templates/spark-env.sh    |  2 +-
 5 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 76e3eedf..76715dc8 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -148,6 +148,8 @@ hcatalog::webhcat::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
 
 # spark
 spark::common::master_host: "%{hiera('bigtop::hadoop_head_node')}"
+spark::common::spark_sql_warehouse_dir: "/user/spark/spark-warehouse"
+spark::common::spark_hive_server2_thrift_port: 12000
 # to enable spark HA, ensure zookeeper is available and uncomment the line below
 #spark::common::zookeeper_connection_string: "%{hiera('hadoop::zk')}"
 
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index b016d6f6..c432036b 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -69,6 +69,7 @@ $roles_map = {
     worker => ["spark-on-yarn"],
     client => ["spark-client"],
     library => ["spark-yarn-slave"],
+    gateway_server => ["spark-thriftserver"],
   },
   spark-standalone => {
     master => ["spark-master"],
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index 2d4a3da2..d2e403ce 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -39,6 +39,31 @@ class spark {
     if ('spark-history-server' in $roles) {
       include spark::history_server
     }
+
+    if ('spark-thriftserver' in $roles) {
+      include spark::spark_thriftserver
+    }
+    
+  }
+
+  class spark_thriftserver {
+    include spark::common
+    
+    package { 'spark-thriftserver': 
+      ensure => latest,
+    }
+
+    service { 'spark-thriftserver':
+      ensure     => running,
+      subscribe  => [
+        Package['spark-thriftserver'],
+        File['/etc/spark/conf/spark-env.sh'],
+        File['/etc/spark/conf/spark-defaults.conf'],
+      ],
+      hasrestart => true,
+      hasstatus  => true,
+    }
+    
   }
 
   class client {
@@ -182,6 +207,8 @@ class spark {
   }
 
   class common(
+      $spark_hive_server2_thrift_port = undef,
+      $spark_sql_warehouse_dir = undef,
       $master_url = undef,
       $master_host = $fqdn,
       $zookeeper_connection_string = undef,
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
index 2ef4a041..ff1cff77 100644
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-defaults.conf
@@ -16,7 +16,7 @@
 <% if @master_url -%>
 spark.master <%= @master_url %>
 <% else -%>
-<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+<% if (scope['spark::deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
 spark.master spark://<%= @master_host %>:<%= @master_port %>
 <% else -%>
 spark.master yarn
@@ -36,3 +36,9 @@ spark.executor.extraLibraryPath <%= @extra_lib_dirs %>
 <% end -%>
 spark.driver.memory <%= @driver_mem %>
 spark.executor.memory <%= @executor_mem %>
+<% if @spark_sql_warehouse_dir -%>
+spark.sql.warehouse.dir <%= @spark_sql_warehouse_dir %>
+<% end -%>
+<% if @spark_hive_server2_thrift_port -%>
+spark.hive.server2.thrift.port <%= @spark_hive_server2_thrift_port %>
+<% end -%>
\ No newline at end of file
diff --git a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
index e1dd40b8..c1041417 100755
--- a/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
+++ b/bigtop-deploy/puppet/modules/spark/templates/spark-env.sh
@@ -27,7 +27,7 @@ export SPARK_MASTER_IP=$STANDALONE_SPARK_MASTER_HOST
 <% if @master_url -%>
 export SPARK_MASTER_URL=<%= @master_url %>
 <% else -%>
-<% if (scope['deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
+<% if (scope['spark::deploy::roles'] & ['spark-master', 'spark-worker']) != [] -%>
 export SPARK_MASTER_URL=spark://<%= @master_host %>:<%= @master_port %>
 <% else -%>
 export SPARK_MASTER_URL=yarn