You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ja...@apache.org on 2014/12/08 01:12:09 UTC
bigtop git commit: BIGTOP-1562. Critical tachyon puppet improvements
Repository: bigtop
Updated Branches:
refs/heads/master eff797efd -> 3f64dfdcc
BIGTOP-1562. Critical tachyon puppet improvements
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3f64dfdc
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3f64dfdc
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3f64dfdc
Branch: refs/heads/master
Commit: 3f64dfdcc88d25adbc1de4c3b2eadd0b7dbeca9f
Parents: eff797e
Author: jayunit100 <ja...@gmail.com>
Authored: Sat Dec 6 12:08:16 2014 -0500
Committer: jayunit100 <ja...@apache.org>
Committed: Sun Dec 7 19:09:19 2014 -0500
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 13 +++-
.../puppet/modules/tachyon/manifests/init.pp | 29 ++++++++-
.../modules/tachyon/templates/log4j.properties | 41 +++++++++++++
.../modules/tachyon/templates/tachyon-env.sh | 64 ++++++++++++++++++++
bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 9 +++
.../vm/vagrant-puppet/vagrantconfig.yaml | 6 +-
6 files changed, 153 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index e7f868f..a2731f5 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -100,6 +100,7 @@ class hadoop_cluster_node {
$giraph_zookeeper_quorum = $hadoop_head_node
$spark_master_host = $hadoop_head_node
+ $tachyon_master_host = $hadoop_head_node
$hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
@@ -194,6 +195,13 @@ class hadoop_worker_node inherits hadoop_cluster_node {
master_ui_port => $spark_master_ui_port,
}
}
+
+ if ($components[0] == undef or "tachyon" in $components) {
+ tachyon::worker { "tachyon worker":
+ master_host => $tachyon_master_host
+ }
+ }
+
}
class hadoop_head_node inherits hadoop_worker_node {
@@ -291,9 +299,8 @@ if ($hadoop_security_authentication == "kerberos") {
}
if ($components[0] == undef or "tachyon" in $components) {
- tachyon::master { "tachyon master":
- master_host => $tachyon_master_host,
- master_port => $tachyon_master_port
+ tachyon::master { "tachyon-master":
+ master_host => $tachyon_master_host
}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
index 8af9b68..55fb34a 100644
--- a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
@@ -14,11 +14,30 @@ class tachyon {
package { "tachyon":
ensure => latest,
}
+
+ # add logging into /var/log/..
+ file {
+ "/etc/tachyon/log4j.properties":
+ content => template("tachyon/log4j.properties"),
+ require => [Package["tachyon"]]
+ }
+
+ # add tachyon-env.sh to point to tachyon master
+ file { "/etc/tachyon/tachyon-env.sh":
+ content => template("tachyon/tachyon-env.sh"),
+ require => [Package["tachyon"]]
+ }
}
- define master($master_host, $master_port) {
+ define master($master_host) {
include common
+ exec {
+ "tachyon formatting":
+ command => "/usr/lib/tachyon/bin/tachyon format",
+ require => [ Package["tachyon"]]
+ }
+
if ( $fqdn == $master_host ) {
service { "tachyon-master":
ensure => running,
@@ -27,17 +46,21 @@ class tachyon {
hasstatus => true,
}
}
+
}
- define worker($master_host, $master_port) {
+ define worker($master_host) {
include common
if ( $fqdn == $master_host ) {
+ notice("tachyon ---> master host")
# We want master to run first in all cases
Service["tachyon-master"] ~> Service["tachyon-worker"]
- }
+ }
+
service { "tachyon-worker":
ensure => running,
+ require => [Package["tachyon"]],
hasrestart => true,
hasstatus => true,
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties b/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties
new file mode 100644
index 0000000..ae84434
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties
@@ -0,0 +1,41 @@
+# May get overridden by System Property
+tachyon.logger.type=Console
+
+log4j.rootLogger=INFO, ${tachyon.logger.type}
+
+log4j.appender.Console=org.apache.log4j.ConsoleAppender
+log4j.appender.Console.Target=System.out
+log4j.appender.Console.layout=org.apache.log4j.PatternLayout
+log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} (%F:%M) - %m%n
+
+# Appender for Master
+log4j.appender.MASTER_LOGGER=tachyon.Log4jFileAppender
+log4j.appender.MASTER_LOGGER.File=/var/log/tachyon/master.log
+
+log4j.appender.MASTER_LOGGER.MaxFileSize=10
+log4j.appender.MASTER_LOGGER.MaxBackupIndex=100
+log4j.appender.MASTER_LOGGER.DeletionPercentage=10
+log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
+#log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Appender for Workers
+log4j.appender.WORKER_LOGGER=tachyon.Log4jFileAppender
+log4j.appender.WORKER_LOGGER.File=/var/log/tachyon/worker.log
+
+log4j.appender.WORKER_LOGGER.MaxFileSize=10
+log4j.appender.WORKER_LOGGER.MaxBackupIndex=100
+log4j.appender.WORKER_LOGGER.DeletionPercentage=10
+log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
+#log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Appender for User
+log4j.appender.USER_LOGGER=tachyon.Log4jFileAppender
+log4j.appender.USER_LOGGER.File=/var/log/tachyon/logs/user.log
+log4j.appender.USER_LOGGER.MaxFileSize=10
+log4j.appender.USER_LOGGER.MaxBackupIndex=10
+log4j.appender.USER_LOGGER.DeletionPercentage=20
+log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout
+log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
+#log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh b/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh
new file mode 100755
index 0000000..1094976
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and
+# edit that to configure Tachyon for your site. At a minimum,
+# the following variables should be set:
+#
+# - JAVA_HOME, to point to your JAVA installation
+# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname
+# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address.
+# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker
+# - TACHYON_RAM_FOLDER, to set where worker stores in memory data
+# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem,
+# org.apache.hadoop.hdfs.DistributedFileSystem)
+
+# The following gives an example:
+
+if [[ `uname -a` == Darwin* ]]; then
+ # Assuming Mac OS X
+ export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home)}
+ export TACHYON_RAM_FOLDER=/Volumes/ramdisk
+ export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
+else
+ # Assuming Linux
+ if [ -z "$JAVA_HOME" ]; then
+ export JAVA_HOME=/usr/lib/jvm/java-7-oracle
+ fi
+ export TACHYON_RAM_FOLDER=/mnt/ramdisk
+fi
+
+export JAVA="$JAVA_HOME/bin/java"
+
+echo "Starting tachyon w/ java = $JAVA "
+
+export TACHYON_MASTER_ADDRESS=<%= @tachyon_master_address %>
+export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underfs
+#export TACHYON_UNDERFS_ADDRESS=hdfs://localhost:9000
+export TACHYON_WORKER_MEMORY_SIZE=1GB
+export TACHYON_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem
+
+echo "TACHYON master => $TACHYON_MASTER_ADDRESS "
+
+CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export TACHYON_JAVA_OPTS+="
+ -Dlog4j.configuration=file:$CONF_DIR/log4j.properties
+ -Dtachyon.debug=false
+ -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS
+ -Dtachyon.underfs.hdfs.impl=$TACHYON_UNDERFS_HDFS_IMPL
+ -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data
+ -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers
+ -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE
+ -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/
+ -Dtachyon.master.worker.timeout.ms=60000
+ -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS
+ -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/
+ -Dorg.apache.jasper.compiler.disablejsr199=true
+ -Djava.net.preferIPv4Stack=true
+"
+
+# Master specific parameters. Default to TACHYON_JAVA_OPTS.
+export TACHYON_MASTER_JAVA_OPTS="$TACHYON_JAVA_OPTS"
+
+# Worker specific parameters that will be shared to all workers. Default to TACHYON_JAVA_OPTS.
+export TACHYON_WORKER_JAVA_OPTS="$TACHYON_JAVA_OPTS"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
index d5eca4a..05a66c2 100755
--- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
@@ -66,6 +66,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# the current directory has puppet recipes which we need for provisioning.
bigtop.vm.synced_folder "../../../", "/bigtop-home"
+ # We also add the bigtop-home output/ dir, so that locally built rpms will be available.
+ puts "Adding rpms from /output/ for local testing."
+
+ # This is a update to allow dev packages
+ # Force success - not worried if this step fails, since we generally only use it for development.
+ bigtop.vm.provision "shell", inline: "sudo echo \"gpgcheck=0\" >> /etc/yum.conf"
+ bigtop.vm.provision "shell", inline: "sudo yum-config-manager --add-repo file:///bigtop-home/output ; true"
+
+ # carry on w/ installation
bigtop.vm.provision "shell", path: "../utils/setup-env.sh"
# shell provisioner to setup basic vm environment
http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml
index d045ce2..4306dbb 100644
--- a/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml
@@ -1,6 +1,6 @@
-memory_size: 1024
+memory_size: 6024
number_cpus: 1
box: "puppetlabs/centos-6.5-64-nocm"
repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
-num_instances: 3
-components: "hadoop,hbase,yarn,mapred-app"
+num_instances: 1
+components: "hadoop,yarn,mapred-app,tachyon"