You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2019/01/18 09:59:40 UTC
[bigtop] branch master updated: BIGTOP-3134. [Puppet] An upgrade to
deploy Alluxio 1.8.1 (#443)
This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 72c6fd8 BIGTOP-3134. [Puppet] An upgrade to deploy Alluxio 1.8.1 (#443)
72c6fd8 is described below
commit 72c6fd85a37865fb93a8bb76002bdc4a1f189a3e
Author: Evans Ye <ev...@apache.org>
AuthorDate: Fri Jan 18 17:59:36 2019 +0800
BIGTOP-3134. [Puppet] An upgrade to deploy Alluxio 1.8.1 (#443)
---
.../puppet/modules/alluxio/manifests/init.pp | 25 +++----
.../modules/alluxio/templates/alluxio-env.sh | 78 ----------------------
.../alluxio/templates/alluxio-site.properties | 40 +++++++++++
.../modules/alluxio/templates/log4j.properties | 77 ---------------------
4 files changed, 49 insertions(+), 171 deletions(-)
diff --git a/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp b/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
index 9c3c10f..6399b80 100644
--- a/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
@@ -21,26 +21,19 @@ class alluxio {
}
}
- class common ($master_host){
+ class common ($master_host, $hdfs_ha = hiera('hadoop::common_hdfs::ha', undef)) {
package { "alluxio":
ensure => latest,
}
exec { "daemon-reload":
- command => "/usr/bin/systemctl daemon-reload",
+ path => ["/bin", "/usr/bin"],
+ command => "systemctl daemon-reload",
require => [ Package["alluxio"] ]
}
- # add logging into /var/log/..
- file {
- "/etc/alluxio/conf/log4j.properties":
- content => template("alluxio/log4j.properties"),
- require => [Package["alluxio"]]
- }
-
- # add alluxio-env.sh to point to alluxio master
- file { "/etc/alluxio/conf/alluxio-env.sh":
- content => template("alluxio/alluxio-env.sh"),
+ file { "/etc/alluxio/conf/alluxio-site.properties":
+ content => template("alluxio/alluxio-site.properties"),
require => [Package["alluxio"]]
}
}
@@ -51,14 +44,14 @@ class alluxio {
exec {
"alluxio formatting":
command => "/usr/lib/alluxio/bin/alluxio format",
- require => [ Package["alluxio"], File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ]
+ require => [ Package["alluxio"], File["/etc/alluxio/conf/alluxio-site.properties"] ]
}
if ( $fqdn == $alluxio::common::master_host ) {
service { "alluxio-master":
ensure => running,
require => [ Package["alluxio"], Exec["daemon-reload"], Exec["alluxio formatting"] ],
- subscribe => [ File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
+ subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
hasrestart => true,
hasstatus => true,
}
@@ -77,8 +70,8 @@ class alluxio {
service { "alluxio-worker":
ensure => running,
- require => [ Package["alluxio"], Exec["daemon-reload"], File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
- subscribe => [ File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
+ require => [ Package["alluxio"], Exec["daemon-reload"], File["/etc/alluxio/conf/alluxio-site.properties"] ],
+ subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
hasrestart => true,
hasstatus => true,
}
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh b/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh
deleted file mode 100755
index 27c7fb2..0000000
--- a/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This file contains environment variables required to run Alluxio. Copy it as alluxio-env.sh and
-# edit that to configure Alluxio for your site. At a minimum,
-# the following variables should be set:
-#
-# - JAVA_HOME, to point to your JAVA installation
-# - ALLUXIO_MASTER_ADDRESS, to bind the master to a different IP address or hostname
-# - ALLUXIO_UNDERFS_ADDRESS, to set the under filesystem address.
-# - ALLUXIO_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker
-# - ALLUXIO_RAM_FOLDER, to set where worker stores in memory data
-# - ALLUXIO_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem,
-# org.apache.hadoop.hdfs.DistributedFileSystem)
-
-# The following gives an example:
-
-if [[ `uname -a` == Darwin* ]]; then
- # Assuming Mac OS X
- export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home)}
- export ALLUXIO_RAM_FOLDER=/Volumes/ramdisk
- export ALLUXIO_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
-else
- # Assuming Linux
- if [ -z "$JAVA_HOME" ]; then
- export JAVA_HOME=/usr/lib/jvm/java-7-oracle
- fi
- export ALLUXIO_RAM_FOLDER=/mnt/ramdisk
-fi
-
-export JAVA="$JAVA_HOME/bin/java"
-
-echo "Starting alluxio w/ java = $JAVA "
-
-export ALLUXIO_MASTER_ADDRESS=<%= @master_host %>
-export ALLUXIO_UNDERFS_ADDRESS=$ALLUXIO_HOME/underfs
-#export ALLUXIO_UNDERFS_ADDRESS=hdfs://localhost:9000
-export ALLUXIO_WORKER_MEMORY_SIZE=1GB
-export ALLUXIO_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem
-
-echo "ALLUXIO master => $ALLUXIO_MASTER_ADDRESS "
-
-CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-export ALLUXIO_JAVA_OPTS+="
- -Dlog4j.configuration=file:$CONF_DIR/log4j.properties
- -Dalluxio.debug=false
- -Dalluxio.underfs.address=$ALLUXIO_UNDERFS_ADDRESS
- -Dalluxio.underfs.hdfs.impl=$ALLUXIO_UNDERFS_HDFS_IMPL
- -Dalluxio.data.folder=$ALLUXIO_UNDERFS_ADDRESS/tmp/alluxio/data
- -Dalluxio.workers.folder=$ALLUXIO_UNDERFS_ADDRESS/tmp/alluxio/workers
- -Dalluxio.worker.memory.size=$ALLUXIO_WORKER_MEMORY_SIZE
- -Dalluxio.worker.data.folder=$ALLUXIO_RAM_FOLDER/alluxioworker/
- -Dalluxio.master.worker.timeout.ms=60000
- -Dalluxio.master.hostname=$ALLUXIO_MASTER_ADDRESS
- -Dalluxio.master.journal.folder=$ALLUXIO_HOME/journal/
- -Dorg.apache.jasper.compiler.disablejsr199=true
- -Djava.net.preferIPv4Stack=true
-"
-
-# Master specific parameters. Default to ALLUXIO_JAVA_OPTS.
-export ALLUXIO_MASTER_JAVA_OPTS="$ALLUXIO_JAVA_OPTS"
-
-# Worker specific parameters that will be shared to all workers. Default to ALLUXIO_JAVA_OPTS.
-export ALLUXIO_WORKER_JAVA_OPTS="$ALLUXIO_JAVA_OPTS"
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties b/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties
new file mode 100644
index 0000000..99fefd8
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Site specific configuration properties for Alluxio
+# Details about all configuration properties http://www.alluxio.org/documentation/en/Configuration-Settings.html
+
+# Common properties
+alluxio.master.hostname=<%= @master_host %>
+<% if defined?(@hdfs_ha) and @hdfs_ha != "disabled" -%>
+alluxio.underfs.address=<%= scope().call_function('hiera',['hadoop_cluster_node::hadoop_namenode_uri']) %>/underFSStorage
+<% else -%>
+alluxio.underfs.address=hdfs://<%= scope().call_function('hiera',['hadoop::common_hdfs::hadoop_namenode_host']) %>:<%= scope().call_function('hiera', ['hadoop::common_hdfs::hadoop_namenode_port']) %>/underFSStorage
+<% end -%>
+
+# Security properties
+# alluxio.security.authorization.permission.enabled=true
+# alluxio.security.authentication.type=SIMPLE
+
+# Worker properties
+# alluxio.worker.memory.size=1GB
+# alluxio.worker.tieredstore.levels=1
+# alluxio.worker.tieredstore.level0.alias=MEM
+# alluxio.worker.tieredstore.level0.dirs.path=/mnt/ramdisk
+
+# User properties
+# alluxio.user.file.readtype.default=CACHE_PROMOTE
+# alluxio.user.file.writetype.default=MUST_CACHE
+
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties b/bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties
deleted file mode 100644
index 00b402e..0000000
--- a/bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# May get overridden by System Property
-alluxio.logger.type=Console
-
-log4j.rootLogger=INFO, ${alluxio.logger.type}
-
-log4j.appender.Console=org.apache.log4j.ConsoleAppender
-log4j.appender.Console.Target=System.out
-log4j.appender.Console.layout=org.apache.log4j.PatternLayout
-log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} (%F:%M) - %m%n
-
-# Appender for Master
-log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log
-log4j.appender.MASTER_LOGGER.MaxFileSize=10MB
-log4j.appender.MASTER_LOGGER.MaxBackupIndex=100
-log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Master Access Log
-log4j.appender.MASTER_ACCESS_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.MASTER_ACCESS_LOGGER.File=${alluxio.logs.dir}/master_access.log
-log4j.appender.MASTER_ACCESS_LOGGER.MaxFileSize=10MB
-log4j.appender.MASTER_ACCESS_LOGGER.MaxBackupIndex=100
-log4j.appender.MASTER_ACCESS_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.MASTER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.MASTER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Workers
-log4j.appender.WORKER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.WORKER_LOGGER.File=${alluxio.logs.dir}/worker.log
-log4j.appender.WORKER_LOGGER.MaxFileSize=10MB
-log4j.appender.WORKER_LOGGER.MaxBackupIndex=100
-log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Worker Access Log
-log4j.appender.WORKER_ACCESS_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.WORKER_ACCESS_LOGGER.File=${alluxio.logs.dir}/worker_access.log
-log4j.appender.WORKER_ACCESS_LOGGER.MaxFileSize=10MB
-log4j.appender.WORKER_ACCESS_LOGGER.MaxBackupIndex=100
-log4j.appender.WORKER_ACCESS_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.WORKER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.WORKER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for User
-log4j.appender.USER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.USER_LOGGER.File=${alluxio.logs.dir}/user.log
-log4j.appender.USER_LOGGER.MaxFileSize=10MB
-log4j.appender.USER_LOGGER.MaxBackupIndex=10
-log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Fuse
-log4j.appender.FUSE_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.FUSE_LOGGER.File=${alluxio.logs.dir}/fuse.log
-log4j.appender.FUSE_LOGGER.MaxFileSize=10MB
-log4j.appender.FUSE_LOGGER.MaxBackupIndex=10
-log4j.appender.FUSE_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.FUSE_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n