You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/05/01 23:52:27 UTC
[6/6] git commit: BIGTOP-958. update deployment code for HCatalog
BIGTOP-958. update deployment code for HCatalog
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/39af29da
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/39af29da
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/39af29da
Branch: refs/heads/master
Commit: 39af29da43fca8513a1ccca8aa01bc276c62d13a
Parents: a36d79e
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 11:30:38 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 14:52:08 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 8 +-
.../puppet/modules/hcatalog/manifests/init.pp | 54 ++++
.../modules/hcatalog/templates/hcatalog-server | 8 +
.../puppet/modules/hcatalog/templates/webhcat.xml | 228 +++++++++++++++
.../puppet/modules/hcatalog/tests/init.pp | 17 +
5 files changed, 314 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 605c6a9..11aeb24 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -229,12 +229,18 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
-
hadoop-sqoop::server { "sqoop server":
}
hadoop-sqoop::client { "sqoop client":
}
+ hcatalog::server { "hcatalog server":
+ kerberos_realm => $kerberos_realm,
+ }
+ hcatalog::webhcat::server { "webhcat server":
+ kerberos_realm => $kerberos_realm,
+ }
+
hue::server { "hue server":
rm_url => $hadoop_rm_url,
rm_proxy_url => $hadoop_rm_proxy_url,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
new file mode 100644
index 0000000..f9c07aa
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hcatalog {
+ define server($port = "9083", $kerberos_realm = "") {
+ package { "hcatalog-server":
+ ensure => latest,
+ }
+
+ file { "/etc/default/hcatalog-server":
+ content => template("hcatalog/hcatalog-server"),
+ require => Package["hcatalog-server"],
+ }
+
+ service { "hcatalog-server":
+ ensure => running,
+ require => [ Package["hcatalog-server"], File["/etc/default/hcatalog-server"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+
+ class webhcat {
+ define server($port = "50111", $kerberos_realm = "") {
+ package { "webhcat-server":
+ ensure => latest,
+ }
+
+ file { "/etc/webhcat/conf/webhcat.xml":
+ content => template("hcatalog/webhcat.xml"),
+ require => Package["webhcat-server"],
+ }
+
+ service { "webhcat-server":
+ ensure => running,
+ require => [ Package["webhcat-server"], File["/etc/webhcat/conf/webhcat.xml"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
new file mode 100644
index 0000000..7291c51
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
@@ -0,0 +1,8 @@
+export METASTORE_PORT=<%= @port %>
+
+export HADOOP_PREFIX=/usr
+export HIVE_HOME=/usr/lib/hive
+export HCAT_LOG_DIR=/var/log/hcatalog
+export HCAT_PID_DIR=/var/run/hcatalog
+#FIXME: workaround for HIVE-2757 (NOTE: HADOOP_HOME is obsolete for Hadoop .23)
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
new file mode 100644
index 0000000..42c5aee
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
@@ -0,0 +1,228 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- The default settings for Templeton. -->
+<!-- Edit templeton-site.xml to change settings for your local -->
+<!-- install. -->
+
+<configuration>
+
+ <property>
+ <name>templeton.port</name>
+ <value><%= @port %></value>
+ <description>The HTTP port for the main server.</description>
+ </property>
+
+ <property>
+ <name>templeton.hadoop.conf.dir</name>
+ <value>${env.HADOOP_CONF_DIR}</value>
+ <description>The path to the Hadoop configuration.</description>
+ </property>
+
+ <property>
+ <name>templeton.jar</name>
+ <value>${env.TEMPLETON_HOME}/share/webhcat/svr/webhcat-0.5.0-SNAPSHOT.jar</value>
+ <description>The path to the Templeton jar file.</description>
+ </property>
+
+ <property>
+ <name>templeton.libjars</name>
+ <value>${env.TEMPLETON_HOME}/share/webhcat/svr/lib/zookeeper-3.4.3.jar</value>
+ <description>Jars to add to the classpath.</description>
+ </property>
+
+ <property>
+ <name>templeton.override.jars</name>
+ <value>hdfs:///user/templeton/ugi.jar</value>
+ <description>
+ Jars to add the the HADOOP_CLASSPATH for all Map Reduce jobs.
+ This is a list of jars that must exist on hdfs that are added
+ to the Distributed Cache.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.override.enabled</name>
+ <value>false</value>
+ <description>
+ Enable the override path in templeton.override.jars
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.streaming.jar</name>
+ <value>hdfs:///user/templeton/hadoop-streaming.jar</value>
+ <description>The hdfs path to the Hadoop streaming jar file.</description>
+ </property>
+
+ <property>
+ <name>templeton.hadoop</name>
+ <value>${env.HADOOP_PREFIX}/bin/hadoop</value>
+ <description>The path to the Hadoop executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.archive</name>
+ <value>hdfs:///apps/templeton/pig-0.10.1.tar.gz</value>
+ <description>The path to the Pig archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.path</name>
+ <value>pig-0.10.1.tar.gz/pig-0.10.1/bin/pig</value>
+ <description>The path to the Pig executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hcat</name>
+ <value>${env.HCAT_PREFIX}/bin/hcat</value>
+ <description>The path to the hcatalog executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.archive</name>
+ <value>hdfs:///apps/templeton/hive-0.10.0.tar.gz</value>
+ <description>The path to the Hive archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.path</name>
+ <value>hive-0.10.0.tar.gz/hive-0.10.0/bin/hive</value>
+ <description>The path to the Hive executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.properties</name>
+ <value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
+ <description>Properties to set when running hive.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.encoding</name>
+ <value>UTF-8</value>
+ <description>The encoding of the stdout and stderr data.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.timeout</name>
+ <value>10000</value>
+ <description>
+ How long in milliseconds a program is allowed to run on the
+ Templeton box.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.exec.max-procs</name>
+ <value>16</value>
+ <description>The maximum number of processes allowed to run at once.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.max-output-bytes</name>
+ <value>1048576</value>
+ <description>
+ The maximum number of bytes from stdout or stderr stored in ram.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.controller.mr.child.opts</name>
+ <value>-server -Xmx256m -Djava.net.preferIPv4Stack=true</value>
+ <description>Java options to be passed to templeton controller map task.
+ The default value of mapreduce child "-Xmx" (heap memory limit)
+ might be close to what is allowed for a map task.
+ Even if templeton controller map task does not need much
+ memory, the jvm (with -server option?)
+ allocates the max memory when it starts. This along with the
+ memory used by pig/hive client it starts can end up exceeding
+ the max memory configured to be allowed for a map task
+ Use this option to set -Xmx to lower value
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.exec.envs</name>
+ <value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
+ <description>The environment variables passed through to exec.</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.hosts</name>
+ <value>127.0.0.1:2181</value>
+ <description>ZooKeeper servers, as comma separated host:port pairs</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.session-timeout</name>
+ <value>30000</value>
+ <description>ZooKeeper session timeout in milliseconds</description>
+ </property>
+
+ <property>
+ <name>templeton.callback.retry.interval</name>
+ <value>10000</value>
+ <description>How long to wait between callback retry attempts in milliseconds</description>
+ </property>
+
+ <property>
+ <name>templeton.callback.retry.attempts</name>
+ <value>5</value>
+ <description>How many times to retry the callback</description>
+ </property>
+
+ <property>
+ <name>templeton.storage.class</name>
+ <value>org.apache.hcatalog.templeton.tool.HDFSStorage</value>
+ <description>The class to use as storage</description>
+ </property>
+
+ <property>
+ <name>templeton.storage.root</name>
+ <value>/templeton-hadoop</value>
+ <description>The path to the directory to use for storage</description>
+ </property>
+
+ <property>
+ <name>templeton.hdfs.cleanup.interval</name>
+ <value>43200000</value>
+ <description>The maximum delay between a thread's cleanup checks</description>
+ </property>
+
+ <property>
+ <name>templeton.hdfs.cleanup.maxage</name>
+ <value>604800000</value>
+ <description>The maximum age of a templeton job</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.cleanup.interval</name>
+ <value>43200000</value>
+ <description>The maximum delay between a thread's cleanup checks</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.cleanup.maxage</name>
+ <value>604800000</value>
+ <description>The maximum age of a templeton job</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
new file mode 100644
index 0000000..59920ae
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hcatalog::server { "test-hcatalog-server": }
+hcatalog::webhcat::server { "test-hcatalog-webhcat-server": }