[bigtop] branch master updated: BIGTOP-3171. Update Kafka Puppet module for version 1.1.1 (#477)

[ev...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new 6d5705e  BIGTOP-3171. Update Kafka Puppet module for version 1.1.1 (#477)
6d5705e is described below

commit 6d5705e3e023caae39410888713f7bbf77578e09
Author: Evans Ye <ev...@apache.org>
AuthorDate: Tue Mar 5 12:49:22 2019 +0800

    BIGTOP-3171. Update Kafka Puppet module for version 1.1.1 (#477)
---
 bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml             | 2 ++
 bigtop-deploy/puppet/modules/kafka/manifests/init.pp           | 2 ++
 bigtop-deploy/puppet/modules/kafka/templates/server.properties | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 0bf81e6..26310d4 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -191,6 +191,8 @@ hadoop::common::tez_jars: "/usr/lib/tez"
 #kafka
 kafka::server::port: "9092"
 kafka::server::zookeeper_connection_string: "%{hiera('bigtop::hadoop_head_node')}:2181"
+# Set to 3 for production deployment
+# kafka::server::offsets_topic_replication_factor: 3
 
 zeppelin::server::spark_master_url: "yarn-client"
 zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}"
diff --git a/bigtop-deploy/puppet/modules/kafka/manifests/init.pp b/bigtop-deploy/puppet/modules/kafka/manifests/init.pp
index f13dec1..0c26fef 100644
--- a/bigtop-deploy/puppet/modules/kafka/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/kafka/manifests/init.pp
@@ -27,6 +27,8 @@ class kafka {
       $bind_addr = undef,
       $port = "9092",
       $zookeeper_connection_string = "localhost:2181",
+      # Default to 1 for less than 3 nodes deployment to work.
+      $offsets_topic_replication_factor = 1,
     ) {
 
     package { 'kafka':
diff --git a/bigtop-deploy/puppet/modules/kafka/templates/server.properties b/bigtop-deploy/puppet/modules/kafka/templates/server.properties
index a58c7b3..30f9efb 100644
--- a/bigtop-deploy/puppet/modules/kafka/templates/server.properties
+++ b/bigtop-deploy/puppet/modules/kafka/templates/server.properties
@@ -125,6 +125,10 @@ log.retention.check.interval.ms=60000
 # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
 log.cleaner.enable=false
 
+# The replication factor for the offsets topic (set higher to ensure availability).
+# Internal topic creation will fail until the cluster size meets this replication factor requirement.
+offsets.topic.replication.factor=<%= @offsets_topic_replication_factor %>
+
 ############################# Zookeeper #############################
 
 # Zookeeper connection string (see zookeeper docs for details).