You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2012/10/15 07:55:57 UTC
svn commit: r1398196 [6/11] - in
/incubator/ambari/branches/branch-0.9-h2-dev: ./ hmc/js/ hmc/package/rpm/
hmc/php/conf/ hmc/php/db/ hmc/php/frontend/ hmc/php/orchestrator/
hmc/php/puppet/ hmc/php/puppet/genmanifest/ hmc/php/util/
hmc/puppet/modules/hd...
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-env.sh.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-env.sh.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-env.sh.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+export JAVA_HOME=<%=scope.function_hdp_java_home()%>
+
+# HBase Configuration directory
+export HBASE_CONF_DIR=${HBASE_CONF_DIR:-<%=scope.function_hdp_template_var("hbase_conf_dir")%>}
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:<%=scope.function_hdp_template_var("::hdp2-hadoop::params::conf_dir")%>
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode"
+
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+export HBASE_MASTER_OPTS="-Xmx<%=scope.function_hdp_template_var("hbase_master_heapsize")%>"
+export HBASE_REGIONSERVER_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseParNewGC -Xmn64m -XX:CMSInitiatingOccupancyFraction=70 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:<%=scope.function_hdp_template_var("hbase_log_dir")%>/log/$USER/gc.log-`date +'%Y%m%d%H%M'` -Xms<%=scope.function_hdp_template_var("hbase_regionserver_heapsize")%> -Xmx<%=scope.function_hdp_template_var("hbase_regionserver_heapsize")%> -XX:ErrorFile=<%=scope.function_hdp_template_var("hbase_log_dir")%>/log/$USER/hs_err_pid%p.log"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+export HBASE_LOG_DIR=<%=scope.function_hdp_template_var("hbase_log_dir")%>
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+export HBASE_PID_DIR=<%=scope.function_hdp_template_var("hbase_pid_dir")%>
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-policy.xml.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-policy.xml.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-policy.xml.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-policy.xml.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ </property>
+</configuration>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-site.xml.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-site.xml.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-site.xml.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/hbase-site.xml.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,377 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.rootdir</name>
+ <value>hdfs://<%=scope.function_hdp_host("namenode_host")%>:8020<%=scope.function_hdp_template_var("hbase_hdfs_root_dir")%></value>
+ <description>The directory shared by region servers and into
+ which HBase persists. The URL should be 'fully-qualified'
+ to include the filesystem scheme. For example, to specify the
+ HDFS directory '/hbase' where the HDFS instance's namenode is
+ running at namenode.example.org on port 9000, set this value to:
+ hdfs://namenode.example.org:9000/hbase. By default HBase writes
+ into /tmp. Change this configuration else all data will be lost
+ on machine restart.
+ </description>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ <description>The mode the cluster will be in. Possible values are
+ false for standalone mode and true for distributed mode. If
+ false, startup will run all HBase and ZooKeeper daemons together
+ in the one JVM.
+ </description>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value><%=scope.function_hdp_template_var("hbase_tmp_dir")%></value>
+ <description>Temporary directory on the local filesystem.
+ Change this setting to point to a location more permanent
+ than '/tmp' (The '/tmp' directory is often cleared on
+ machine restart).
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.info.bindAddress</name>
+ <value><%=scope.function_hdp_host("hbase_master_host")%></value>
+ <description>The bind address for the HBase Master web UI
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <value><%=scope.function_hdp_template_var("regionserver_memstore_upperlimit")%></value>
+ <description>Maximum size of all memstores in a region server before new
+ updates are blocked and flushes are forced. Defaults to 40% of heap
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value><%=scope.function_hdp_template_var("regionserver_handlers")%></value>
+ <description>Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value><%=scope.function_hdp_template_var("hregion_majorcompaction")%></value>
+ <description>The time (in miliseconds) between 'major' compactions of all
+ HStoreFiles in a region. Default: 1 day.
+ Set to 0 to disable automated major compactions.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value><%=scope.function_hdp_template_var("regionserver_handlers")%></value>
+ <description>Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value><%=scope.function_hdp_template_var("hregion_majorcompaction")%></value>
+ <description>The time (in miliseconds) between 'major' compactions of all
+ HStoreFiles in a region. Default: 1 day.
+ Set to 0 to disable automated major compactions.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value><%=scope.function_hdp_template_var("regionserver_memstore_lowerlimit")%></value>
+ <description>When memstores are being forced to flush to make room in
+ memory, keep flushing until we hit this mark. Defaults to 35% of heap.
+ This value equal to hbase.regionserver.global.memstore.upperLimit causes
+ the minimum possible flushing to occur when updates are blocked due to
+ memstore limiting.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value><%=scope.function_hdp_template_var("hregion_blockmultiplier")%></value>
+ <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ time hbase.hregion.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value><%=scope.function_hdp_template_var("hregion_memstoreflushsize")%></value>
+ <description>
+ Memstore will be flushed to disk if size of the memstore
+ exceeds this number of bytes. Value is checked by a thread that runs
+ every hbase.server.thread.wakefrequency.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value><%=scope.function_hdp_template_var("hregion_blockmultiplier")%></value>
+ <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ time hbase.hregion.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value><%=scope.function_hdp_template_var("hregion_memstoreflushsize")%></value>
+ <description>
+ Memstore will be flushed to disk if size of the memstore
+ exceeds this number of bytes. Value is checked by a thread that runs
+ every hbase.server.thread.wakefrequency.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value><%=scope.function_hdp_template_var("regionserver_memstore_lab")%></value>
+ <description>
+ Enables the MemStore-Local Allocation Buffer,
+ a feature which works to prevent heap fragmentation under
+ heavy write loads. This can reduce the frequency of stop-the-world
+ GC pauses on large heaps.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value><%=scope.function_hdp_template_var("hstorefile_maxsize")%></value>
+ <description>
+ Maximum HStoreFile size. If any one of a column families' HStoreFiles has
+ grown to exceed this value, the hosting HRegion is split in two.
+ Default: 1G.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value><%=scope.function_hdp_template_var("client_scannercaching")%></value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value><%=scope.function_hdp_template_var("zookeeper_sessiontimeout")%></value>
+ <description>ZooKeeper session timeout.
+ HBase passes this to the zk quorum as suggested maximum time for a
+ session (This setting becomes zookeeper's 'maxSessionTimeout'). See
+ http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ "The client sends a requested timeout, the server responds with the
+ timeout that it can give the client. " In milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value><%=scope.function_hdp_template_var("hfile_max_keyvalue_size")%></value>
+ <description>Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value><%=scope.function_hdp_template_var("client_scannercaching")%></value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value><%=scope.function_hdp_template_var("zookeeper_sessiontimeout")%></value>
+ <description>ZooKeeper session timeout.
+ HBase passes this to the zk quorum as suggested maximum time for a
+ session (This setting becomes zookeeper's 'maxSessionTimeout'). See
+ http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ "The client sends a requested timeout, the server responds with the
+ timeout that it can give the client. " In milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value><%=scope.function_hdp_template_var("hfile_max_keyvalue_size")%></value>
+ <description>Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value><%=scope.function_hdp_template_var("hstore_compactionthreshold")%></value>
+ <description>
+ If more than this number of HStoreFiles in any one HStore
+ (one HStoreFile is written per flush of memstore) then a compaction
+ is run to rewrite all HStoreFiles files as one. Larger numbers
+ put off compaction but when it runs, it takes longer to complete.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value><%=scope.function_hdp_template_var("hstore_blockingstorefiles")%></value>
+ <description>
+ If more than this number of StoreFiles in any one Store
+ (one StoreFile is written per flush of MemStore) then updates are
+ blocked for this HRegion until a compaction is completed, or
+ until hbase.hstore.blockingWaitTime has been exceeded.
+ </description>
+ </property>
+ <property>
+ <name>hfile.block.cache.size</name>
+ <value><%=scope.function_hdp_template_var("hfile_blockcache_size")%></value>
+ <description>
+ Percentage of maximum heap (-Xmx setting) to allocate to block cache
+ used by HFile/StoreFile. Default of 0.25 means allocate 25%.
+ Set to 0 to disable but it's not recommended.
+ </description>
+ </property>
+
+ <!-- The following properties configure authentication information for
+ HBase processes when using Kerberos security. There are no default
+ values, included here for documentation purposes -->
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value><%=scope.function_hdp_template_var("keytab_path")%>/hm.service.keytab</value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HMaster server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value>hm/_HOST@<%=scope.function_hdp_template_var("kerberos_domain")%></value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HMaster process. The principal name should
+ be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname
+ portion, it will be replaced with the actual hostname of the running
+ instance.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value><%=scope.function_hdp_template_var("keytab_path")%>/rs.service.keytab</value>
+ <description>Full path to the kerberos keytab file to use for logging in
+ the configured HRegionServer server principal.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value>rs/_HOST@<%=scope.function_hdp_template_var("kerberos_domain")%></value>
+ <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
+ that should be used to run the HRegionServer process. The principal name
+ should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the
+ hostname portion, it will be replaced with the actual hostname of the
+ running instance. An entry for this principal must exist in the file
+ specified in hbase.regionserver.keytab.file
+ </description>
+ </property>
+
+ <!-- Additional configuration specific to HBase security -->
+ <property>
+ <name>hbase.superuser</name>
+ <value>hbase</value>
+ <description>List of users or groups (comma-separated), who are allowed
+ full privileges, regardless of stored ACLs, across the cluster.
+ Only used when HBase security is enabled.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value><%=scope.function_hdp_template_var("preloaded_regioncoprocessor_classes")%></value>
+ <description>A comma-separated list of Coprocessors that are loaded by
+ default on all tables. For any override coprocessor method, these classes
+ will be called in order. After implementing your own Coprocessor, just put
+ it in HBase's classpath and add the fully qualified class name here.
+ A coprocessor can also be loaded on demand by setting HTableDescriptor.
+ </description>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value><%=scope.function_hdp_template_var("preloaded_mastercoprocessor_classes")%></value>
+ <description>A comma-separated list of
+ org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are
+ loaded by default on the active HMaster process. For any implemented
+ coprocessor methods, the listed classes will be called in order. After
+ implementing your own MasterObserver, just put it in HBase's classpath
+ and add the fully qualified class name here.
+ </description>
+ </property>
+
+ <!--
+ The following three properties are used together to create the list of
+ host:peer_port:leader_port quorum servers for ZooKeeper.
+ -->
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value><%=zkh=scope.function_hdp_host("zookeeper_hosts");scope.function_hdp_is_empty(zkh) ? "" : [zkh].flatten.join(",")%></value>
+ <description>Comma separated list of servers in the ZooKeeper Quorum.
+ For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+ By default this is set to localhost for local and pseudo-distributed modes
+ of operation. For a fully-distributed setup, this should be set to a full
+ list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
+ this is the list of servers which we will start/stop ZooKeeper on.
+ </description>
+ </property>
+ <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
+
+ <property>
+ <name>dfs.support.append</name>
+ <value><%=scope.function_hdp_template_var("hdfs_support_append")%></value>
+ <description>Does HDFS allow appends to files?
+ This is an hdfs config. set in here so the hdfs client will do append support.
+ You must ensure that this config. is true serverside too when running hbase
+ (You will have to restart your cluster after setting it).
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value><%=scope.function_hdp_template_var("hdfs_enable_shortcircuit_read")%></value>
+ <description>Enable/Disable short circuit read for your client.
+ Hadoop servers should be configured to allow short circuit read
+ for the hbase user for this to take effect
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value><%=scope.function_hdp_template_var("hdfs_enable_shortcircuit_skipchecksum")%></value>
+ <description>Enable/disbale skipping the checksum check</description>
+ </property>
+</configuration>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/log4j.properties.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/log4j.properties.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/log4j.properties.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/log4j.properties.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,80 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=INFO
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of .META. messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/regionservers.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/regionservers.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/regionservers.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hbase/templates/regionservers.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,3 @@
+<%h=scope.function_hdp_host("hbase_rs_hosts"); (h.kind_of?(Array) ? h : []).each do |host|-%>
+<%= host %>
+<%end-%>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/hcatSmoke.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/hcatSmoke.sh?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/hcatSmoke.sh (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/hcatSmoke.sh Mon Oct 15 05:55:46 2012
@@ -0,0 +1,24 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+export tablename=$1
+hcat -e "show tables"
+hcat -e "drop table IF EXISTS ${tablename}"
+hcat -e "create table ${tablename} ( id INT, name string ) stored as rcfile ;"
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/pigSmoke.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/pigSmoke.sh?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/pigSmoke.sh (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/files/pigSmoke.sh Mon Oct 15 05:55:46 2012
@@ -0,0 +1,24 @@
+/*
+ *
+ * licensed to the apache software foundation (asf) under one
+ * or more contributor license agreements. see the notice file
+ * distributed with this work for additional information
+ * regarding copyright ownership. the asf licenses this file
+ * to you under the apache license, version 2.0 (the
+ * "license"); you may not use this file except in compliance
+ * with the license. you may obtain a copy of the license at
+ *
+ * http://www.apache.org/licenses/license-2.0
+ *
+ * unless required by applicable law or agreed to in writing,
+ * software distributed under the license is distributed on an
+ * "as is" basis, without warranties or conditions of any
+ * kind, either express or implied. see the license for the
+ * specific language governing permissions and limitations
+ * under the license.
+ *
+ */
+
+A = load 'passwd' using PigStorage(':');
+B = foreach A generate \$0 as id;
+store B into 'pigsmoke.out';
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/hcat/service_check.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/hcat/service_check.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/hcat/service_check.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/hcat/service_check.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,55 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hcat::hcat::service_check()
+{
+ $unique = hdp_unique_id_and_date()
+ $smoke_test_user = $hdp2::params::smokeuser
+ $output_file = "/apps/hive/warehouse/hcatsmoke${unique}"
+
+ $test_cmd = "fs -test -e ${output_file}"
+
+ anchor { 'hdp2-hcat::hcat::service_check::begin':}
+
+ file { '/tmp/hcatSmoke.sh':
+ ensure => present,
+ source => "puppet:///modules/hdp2-hcat/hcatSmoke.sh",
+ mode => '0755',
+ }
+
+ exec { '/tmp/hcatSmoke.sh':
+ command => "su - ${smoke_test_user} -c 'sh /tmp/hcatSmoke.sh hcatsmoke${unique}'",
+ tries => 3,
+ try_sleep => 5,
+ require => File['/tmp/hcatSmoke.sh'],
+ path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ notify => Hdp2-hadoop::Exec-hadoop['hcat::service_check::test'],
+ logoutput => "true"
+ }
+
+ hdp2-hadoop::exec-hadoop { 'hcat::service_check::test':
+ command => $test_cmd,
+ refreshonly => true,
+ require => Exec['/tmp/hcatSmoke.sh'],
+ before => Anchor['hdp2-hcat::hcat::service_check::end']
+ }
+
+ anchor{ 'hdp2-hcat::hcat::service_check::end':}
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/init.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/init.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/init.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,72 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hcat(
+ $service_state = $hdp2::params::cluster_client_state
+) inherits hdp2-hcat::params
+{
+ $hcat_config_dir = $hdp2-hcat::params::hcat_conf_dir
+
+
+ if ($hdp2::params::use_32_bits_on_slaves == false) {
+ $size = 64
+ } else {
+ $size = 32
+ }
+
+ if ($service_state == 'no_op') {
+ } elsif ($service_state == 'uninstalled') {
+ hdp2::package { 'hcat' :
+ ensure => 'uninstalled',
+ size => $size
+ }
+
+ hdp2::directory { $hcat_config_dir:
+ service_state => $service_state,
+ force => true
+ }
+
+ Hdp2::Package['hcat'] -> Hdp2::Directory[$hcat_config_dir]
+
+ } elsif ($service_state == 'installed_and_configured') {
+ hdp2::package { 'hcat' :
+ size => $size
+ }
+
+ hdp2::directory { $hcat_config_dir:
+ service_state => $service_state,
+ force => true
+ }
+
+ hdp2-hcat::configfile { 'hcat-env.sh':}
+
+ Hdp2::Package['hcat'] -> Hdp2::Directory[$hcat_config_dir] -> Hdp-hcat::Configfile<||>
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
+
+### config files
+define hdp2-hcat::configfile()
+{
+ hdp2::configfile { "${hdp2::params::hcat_conf_dir}/${name}":
+ component => 'hcat'
+ }
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/params.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/params.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/manifests/params.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hcat::params() inherits hdp2::params
+{
+ $hcat_conf_dir = $hdp2::params::hcat_conf_dir
+
+ $hcat_metastore_port = hdp_default("hcat_metastore_port",9933)
+ $hcat_lib = hdp_default("hcat_lib","/usr/lib/hcatalog/share/hcatalog") #TODO: should I remove and just use hcat_dbroot
+
+ ### hcat-env
+ $hcat_dbroot = hdp_default("hadoop/hcat-env/hcat_dbroot",$hcat_lib)
+
+ $hcat_log_dir = hdp_default("hadoop/hcat-env/hcat_log_dir","/var/log/hcatalog")
+
+ $hcat_pid_dir = hdp_default("hadoop/hcat-env/hcat_pid_dir","/var/run/hcatalog")
+
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/templates/hcat-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/templates/hcat-env.sh.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/templates/hcat-env.sh.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hcat/templates/hcat-env.sh.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JAVA_HOME=<%=scope.function_hdp_java_home()%>
+HCAT_PID_DIR=<%=scope.function_hdp_template_var("hcat_pid_dir")%>/
+HCAT_LOG_DIR=<%=scope.function_hdp_template_var("hcat_log_dir")%>/
+HCAT_CONF_DIR=<%=scope.function_hdp_template_var("hcat_conf_dir")%>
+HADOOP_HOME=${HADOOP_HOME:-<%=scope.function_hdp_template_var("::hdp2::params::hadoop_home")%>}
+#DBROOT is the path where the connector jars are downloaded
+DBROOT=<%=scope.function_hdp_template_var("hcat_dbroot")%>
+USER=<%=scope.function_hdp_user("hcat_user")%>
+METASTORE_PORT=<%=scope.function_hdp_template_var("hcat_metastore_port")%>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/files/hiveSmoke.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/files/hiveSmoke.sh?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/files/hiveSmoke.sh (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/files/hiveSmoke.sh Mon Oct 15 05:55:46 2012
@@ -0,0 +1,23 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+export tablename=$1
+echo "CREATE EXTERNAL TABLE IF NOT EXISTS ${tablename} ( foo INT, bar STRING );" | hive
+echo "DESCRIBE ${tablename};" | hive
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/client.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/client.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/client.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/client.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,40 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::client(
+ $service_state = $hdp2::params::cluster_client_state,
+ $hive_server_host = undef
+) inherits hdp2::params
+{
+ if ($service_state == 'no_op') {
+ } elsif ($service_state in ['installed_and_configured','uninstalled']) {
+ if ($hdp2::params::service_exists['hdp2-hive::server'] != true) {
+ #installs package, creates user, sets configuration
+ class { 'hdp2-hive':
+ service_state => $service_state
+ }
+ if ($hive_server_host != undef) {
+ Hdp2-Hive::Configfile<||>{hive_server_host => $hive_server_host}
+ }
+ }
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/hive/service_check.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/hive/service_check.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/hive/service_check.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/hive/service_check.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,55 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::hive::service_check()
+{
+ $unique = hdp_unique_id_and_date()
+ $smoke_test_user = $hdp2::params::smokeuser
+ $output_file = "/apps/hive/warehouse/hivesmoke${unique}"
+
+ $test_cmd = "fs -test -e ${output_file}"
+
+ anchor { 'hdp2-hive::hive::service_check::begin':}
+
+ file { '/tmp/hiveSmoke.sh':
+ ensure => present,
+ source => "puppet:///modules/hdp2-hive/hiveSmoke.sh",
+ mode => '0755',
+ }
+
+ exec { '/tmp/hiveSmoke.sh':
+ command => "su - ${smoke_test_user} -c 'sh /tmp/hiveSmoke.sh hivesmoke${unique}'",
+ tries => 3,
+ try_sleep => 5,
+ require => File['/tmp/hiveSmoke.sh'],
+ path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ notify => Hdp2-hadoop::Exec-hadoop['hive::service_check::test'],
+ logoutput => "true"
+ }
+
+ hdp2-hadoop::exec-hadoop { 'hive::service_check::test':
+ command => $test_cmd,
+ refreshonly => true,
+ require => Exec['/tmp/hiveSmoke.sh'],
+ before => Anchor['hdp2-hive::hive::service_check::end']
+ }
+
+ anchor{ 'hdp2-hive::hive::service_check::end':}
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/init.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/init.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/init.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,82 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive(
+ $service_state,
+ $server = false
+)
+{
+ include hdp2-hive::params
+
+ $hive_user = $hdp2-hive::params::hive_user
+ $hive_config_dir = $hdp2-hive::params::hive_conf_dir
+
+ anchor { 'hdp2-hive::begin': }
+ anchor { 'hdp2-hive::end': }
+
+ if ($service_state == 'uninstalled') {
+ hdp2::package { 'hive' :
+ ensure => 'uninstalled'
+ }
+
+ hdp2::directory { $hive_config_dir:
+ service_state => $service_state,
+ force => true
+ }
+
+ Anchor['hdp2-hive::begin'] -> Hdp2::Package['hive'] -> Hdp2::Directory[$hive_config_dir] -> Anchor['hdp2-hive::end']
+
+ } else {
+ hdp2::package { 'hive' : }
+ if ($server == true ) {
+ class { 'hdp2-hive::mysql-connector': }
+ }
+
+ hdp2::user{ $hive_user:}
+
+ hdp2::directory { $hive_config_dir:
+ service_state => $service_state,
+ force => true
+ }
+
+ hdp2-hive::configfile { ['hive-env.sh','hive-site.xml']: }
+
+ Anchor['hdp2-hive::begin'] -> Hdp2::Package['hive'] -> Hdp2::User[$hive_user] ->
+ Hdp2::Directory[$hive_config_dir] -> Hdp2-hive::Configfile<||> -> Anchor['hdp2-hive::end']
+
+ if ($server == true ) {
+ Hdp2::Package['hive'] -> Hdp2::User[$hive_user] -> Class['hdp2-hive::mysql-connector'] -> Anchor['hdp2-hive::end']
+ }
+ }
+}
+
+### config files
+define hdp2-hive::configfile(
+ $mode = undef,
+ $hive_server_host = undef
+)
+{
+ hdp2::configfile { "${hdp2-hive::params::hive_conf_dir}/${name}":
+ component => 'hive',
+ owner => $hdp2-hive::params::hive_user,
+ mode => $mode,
+ hive_server_host => $hive_server_host
+ }
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/mysql-connector.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/mysql-connector.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/mysql-connector.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/mysql-connector.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,45 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::mysql-connector()
+{
+ include hdp2-hive::params
+
+ $hive_lib = $hdp2-hive::params::hive_lib
+ $target = "${hive_lib}/mysql-connector-java.jar"
+
+ anchor { 'hdp2-hive::mysql-connector::begin':}
+
+ hdp2::package { 'mysql-connector' :
+ require => Anchor['hdp2-hive::mysql-connector::begin']
+ }
+
+ hdp2::exec { 'hive mkdir -p ${artifact_dir} ; cp /usr/share/java/mysql-connector-java.jar ${target}':
+ command => "mkdir -p ${artifact_dir} ; cp /usr/share/java/mysql-connector-java.jar ${target}",
+ unless => "test -f ${target}",
+ creates => $target,
+ path => ["/bin","/usr/bin/"],
+ require => Hdp2::Package['mysql-connector'],
+ notify => Anchor['hdp2-hive::mysql-connector::end'],
+ }
+
+ anchor { 'hdp2-hive::mysql-connector::end':}
+
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/params.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/params.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/params.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::params() inherits hdp2::params
+{
+
+ #TODO: will move to globals
+ $hive_metastore_user_name = hdp_default("hadoop/hive-site/hive_metastore_user_name","dbusername")
+ $hive_metastore_user_passwd = hdp_default("hadoop/hive-site/hive_metastore_user_passwd","dbpassword")
+
+ ### users
+ $hive_user = $hdp2::params::hive_user
+
+ ### common
+ $hive_metastore_port = hdp_default("hive_metastore_port",9083)
+ $hive_lib = hdp_default("hive_lib","/usr/lib/hive/lib/") #TODO: should I remove and just use hive_dbroot
+
+ ### hive-env
+ $hive_conf_dir = $hdp2::params::hive_conf_dir
+
+ $hive_dbroot = hdp_default("hadoop/hive-env/hive_dbroot",$hive_lib)
+
+ $hive_log_dir = hdp_default("hadoop/hive-env/hive_log_dir","/var/log/hive")
+
+ $hive_pid_dir = hdp_default("hadoop/hive-env/hive_pid_dir","/var/run/hive")
+
+ ### hive-site
+ $hive_database_name = hdp_default("hadoop/hive-site/hive_database_name","hive")
+
+ $hive_metastore_principal = hdp_default("hadoop/hive-site/hive_metastore_principal")
+
+ $hive_metastore_sasl_enabled = hdp_default("hadoop/hive-site/hive_metastore_sasl_enabled","false")
+
+ #TODO: using instead hive_server_host in hdp2::params $hive_metastore_server_host = hdp_default("hadoop/hive-site/hive_metastore_server_host")
+
+ $keytab_path = hdp_default("hadoop/hive-site/keytab_path")
+
+ ###mysql connector
+ $download_url = $hdp2::params::gpl_artifacts_download_url
+ $mysql_connector_url = "${download_url}/mysql-connector-java-5.1.18.zip"
+ $hive_aux_jars_path = hdp_default("hive_aux_jars_path","/usr/lib/hcatalog/share/hcatalog/")
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/server.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/server.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/server.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/server.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,69 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::server(
+ $service_state = $hdp2::params::cluster_service_state,
+ $opts = {}
+) inherits hdp2-hive::params
+{
+
+ if ($service_state == 'no_op') {
+ } elsif ($service_state in ['running','stopped','installed_and_configured','uninstalled']) {
+
+ $hdp2::params::service_exists['hdp2-hive::server'] = true
+
+ #installs package, creates user, sets configuration
+ class{ 'hdp2-hive' :
+ service_state => $service_state,
+ server => true
+ }
+
+ Hdp2-Hive::Configfile<||>{hive_server_host => $hdp2::params::host_address}
+
+ class { 'hdp2-hive::hdfs-directories' :
+ service_state => $service_state
+ }
+
+ class { 'hdp2-hive::service' :
+ ensure => $service_state
+ }
+
+ #top level does not need anchors
+ Class['hdp2-hive'] -> Class['hdp2-hive::hdfs-directories'] -> Class['hdp2-hive::service']
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
+
+class hdp2-hive::hdfs-directories($service_state)
+{
+ $hive_user = $hdp2-hive::params::hive_user
+
+ hdp2-hadoop::hdfs::directory{ '/apps/hive/warehouse':
+ service_state => $service_state,
+ owner => $hive_user,
+ mode => '777',
+ recursive_chmod => true
+ }
+ hdp2-hadoop::hdfs::directory{ "/user/${hive_user}":
+ service_state => $service_state,
+ owner => $hive_user
+ }
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/service.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/service.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/service.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/manifests/service.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,74 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-hive::service(
+ $ensure
+)
+{
+ include $hdp2-hive::params
+
+ $user = $hdp2-hive::params::hive_user
+ $hadoop_home = $hdp2::params::hadoop_home
+ $hive_log_dir = $hdp2-hive::params::hive_log_dir
+ $cmd = "env HADOOP_HOME=${hadoop_home} nohup hive --service metastore > ${hive_log_dir}/hive.out 2> ${hive_log_dir}/hive.log &"
+ $pid_file = "${hdp2-hive::params::hive_pid_dir}/hive.pid"
+
+ if ($ensure == 'running') {
+ $daemon_cmd = "su - ${user} -c '${cmd} '"
+ $no_op_test = "ls ${pid_file} >/dev/null 2>&1 && ps `cat ${pid_file}` >/dev/null 2>&1"
+ } elsif ($ensure == 'stopped') {
+ #TODO: this needs to be fixed
+ $daemon_cmd = "ps aux | awk '{print \$1,\$2}' | grep \"^${user} \" | awk '{print \$2}' | xargs kill >/dev/null 2>&1"
+ $no_op_test = "ps aux | awk '{print \$1,\$2}' | grep \"^${user} \""
+ } else {
+ $daemon_cmd = undef
+ }
+
+ hdp2-hive::service::directory { $hdp2-hive::params::hive_pid_dir : }
+ hdp2-hive::service::directory { $hdp2-hive::params::hive_log_dir : }
+
+ anchor{'hdp2-hive::service::begin':} -> Hdp2-hive::Service::Directory<||> -> anchor{'hdp2-hive::service::end':}
+
+ if ($daemon_cmd != undef) {
+ if ($ensure == 'running') {
+ hdp2::exec { $daemon_cmd:
+ command => $daemon_cmd,
+ unless => $no_op_test
+ }
+ } elsif ($ensure == 'stopped') {
+ hdp2::exec { $daemon_cmd:
+ command => $daemon_cmd,
+ onlyif => $no_op_test
+ }
+ }
+ Hdp2-hive::Service::Directory<||> -> Hdp2::Exec[$daemon_cmd] -> Anchor['hdp2-hive::service::end']
+ }
+}
+
+define hdp2-hive::service::directory()
+{
+ hdp2::directory_recursive_create { $name:
+ owner => $hdp2-hive::params::hive_user,
+ mode => '0755',
+ service_state => $ensure,
+ force => true
+ }
+}
+
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-env.sh.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-env.sh.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-env.sh.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-env.sh.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set Hive and Hadoop environment variables here. These variables can be used
+# to control the execution of Hive. It should be used by admins to configure
+# the Hive installation (so that users do not have to set environment variables
+# or set command line parameters to get correct behavior).
+#
+# The hive service being invoked (CLI/HWI etc.) is available via the environment
+# variable SERVICE
+
+# Hive Client memory usage can be an issue if a large number of clients
+# are running at the same time. The flags below have been useful in
+# reducing memory usage:
+#
+ if [ "$SERVICE" = "cli" ]; then
+ if [ -z "$DEBUG" ]; then
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit"
+ else
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
+ fi
+ fi
+
+# The heap size of the jvm stared by hive shell script can be controlled via:
+
+export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024}
+
+# Larger heap size may be required when running queries over large number of files or partitions.
+# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
+# appropriate for hive server (hwi etc).
+
+
+# Set HADOOP_HOME to point to a specific hadoop install directory
+HADOOP_HOME=${HADOOP_HOME:-<%=scope.function_hdp_template_var("::hdp2::params::hadoop_home")%>}
+
+# Hive Configuration Directory can be controlled by:
+export HIVE_CONF_DIR=<%=scope.function_hdp_template_var("hive_conf_dir")%>
+
+# Folder containing extra ibraries required for hive compilation/execution can be controlled by:
+# export HIVE_AUX_JARS_PATH=
+export HIVE_AUX_JARS_PATH=<%=scope.function_hdp_template_var("hive_aux_jars_path")%>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-site.xml.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-site.xml.erb?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-site.xml.erb (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-hive/templates/hive-site.xml.erb Mon Oct 15 05:55:46 2012
@@ -0,0 +1,128 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>hive.metastore.local</name>
+ <value>false</value>
+ <description>controls whether to connect to remove metastore server or
+ open a new metastore server in Hive Client JVM</description>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://<%=scope.function_hdp_host("hive_mysql_host")%>/<%=scope.function_hdp_template_var("hive_database_name")%>?createDatabaseIfNotExist=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ <description>Driver class name for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value><%=scope.function_hdp_template_var("hive_metastore_user_name")%></value>
+ <description>username to use against metastore database</description>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value><%=scope.function_hdp_template_var("hive_metastore_user_passwd")%></value>
+ <description>password to use against metastore database</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>/apps/hive/warehouse</value>
+ <description>location of default database for the warehouse</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.sasl.enabled</name>
+ <value><%=scope.function_hdp_template_var("hive_metastore_sasl_enabled")%></value>
+ <description>If true, the metastore thrift interface will be secured with SASL.
+ Clients must authenticate with Kerberos.</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.kerberos.keytab.file</name>
+ <value><%=scope.function_hdp_template_var("keytab_path")%>/hive.service.keytab</value>
+ <description>The path to the Kerberos Keytab file containing the metastore
+ thrift server's service principal.</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.kerberos.principal</name>
+ <value><%=scope.function_hdp_template_var("hive_metastore_principal")%></value>
+ <description>The service principal for the metastore thrift server. The special
+ string _HOST will be replaced automatically with the correct host name.</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.cache.pinobjtypes</name>
+ <value>Table,Database,Type,FieldSchema,Order</value>
+ <description>List of comma separated metastore object types that should be pinned in the cache</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://<%=scope.function_hdp_host("hive_server_host")%>:<%=scope.function_hdp_template_var("hive_metastore_port")%></value>
+ <description>URI for client to contact metastore server</description>
+ </property>
+
+ <property>
+ <name>hive.semantic.analyzer.factory.impl</name>
+ <value>org.apache.hivealog.cli.HCatSemanticAnalyzerFactory</value>
+ <description>controls which SemanticAnalyzerFactory implemenation class is used by CLI</description>
+ </property>
+
+ <property>
+ <name>hadoop.clientside.fs.operations</name>
+ <value>true</value>
+ <description>FS operations are owned by client</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.client.socket.timeout</name>
+ <value>60</value>
+ <description>MetaStore Client socket timeout in seconds</description>
+ </property>
+
+ <property>
+ <name>hive.metastore.execute.setugi</name>
+ <value>true</value>
+ <description>In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. Further note that its best effort. If client sets its to true and server sets it to false, client setting will be ignored.</description>
+ </property>
+
+ <property>
+ <name>hive.security.authorization.enabled</name>
+ <value>true</value>
+ <description>enable or disable the hive client authorization</description>
+ </property>
+
+ <property>
+ <name>hive.security.authorization.manager</name>
+ <value>org.apache.hcatalog.security.HdfsAuthorizationProvider</value>
+ <description>the hive client authorization manager class name.
+ The user defined authorization class should implement interface org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider. </description>
+ </property>
+
+</configuration>
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/files/startMysql.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/files/startMysql.sh?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/files/startMysql.sh (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/files/startMysql.sh Mon Oct 15 05:55:46 2012
@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+mysqldbuser=$1
+mysqldbpasswd=$2
+mysqldbhost=$3
+
+echo "Adding user $mysqldbuser@$mysqldbhost"
+echo "CREATE USER '$mysqldbuser'@'$mysqldbhost' IDENTIFIED BY '$mysqldbpasswd';" | mysql -u root
+echo "GRANT ALL PRIVILEGES ON *.* TO '$mysqldbuser'@'$mysqldbhost';" | mysql -u root
+echo "flush privileges;" | mysql -u root
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/init.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/init.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/init.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,22 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-mysql(){}
+
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/params.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/params.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/params.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,26 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-mysql::params() inherits hdp2-hive::params
+{
+ $db_name = "$hdp2-hive::params::hive_database_name"
+ $db_user = $hdp2-hive::params::hive_metastore_user_name
+ $db_pw = $hdp2-hive::params::hive_metastore_user_passwd
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/server.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/server.pp?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/server.pp (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-mysql/manifests/server.pp Mon Oct 15 05:55:46 2012
@@ -0,0 +1,71 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp2-mysql::server(
+ $service_state = $hdp2::params::cluster_service_state,
+ $opts = {}
+) inherits hdp2-mysql::params
+{
+ if ($service_state in ['no_op','uninstalled']) {
+ } elsif ($service_state in ['running','stopped','installed_and_configured']) {
+
+ $db_user = $hdp2-mysql::params::db_user
+ $db_pw = $hdp2-mysql::params::db_pw
+ $db_name = $hdp2-mysql::params::db_name
+ $host = $hdp2::params::hive_mysql_host
+
+ anchor { 'hdp2-mysql::server::begin':}
+
+ hdp2::package { 'mysql' :
+ size => 32,
+ require => Anchor['hdp2-mysql::server::begin']
+ }
+
+ hdp2::exec { 'mysqld start':
+ command => '/etc/init.d/mysqld start',
+ unless => '/etc/init.d/mysqld status',
+ require => Hdp2::Package['mysql'],
+ notify => File['/tmp/startMysql.sh']
+ }
+
+ file { '/tmp/startMysql.sh':
+ ensure => present,
+ source => "puppet:///modules/hdp2-mysql/startMysql.sh",
+ mode => '0755',
+ require => Hdp2::Exec['mysqld start'],
+ notify => Exec['/tmp/startMysql.sh']
+ }
+
+ exec { '/tmp/startMysql.sh':
+ command => "sh /tmp/startMysql.sh ${db_user} ${db_pw} ${host}",
+ tries => 3,
+ try_sleep => 5,
+ require => File['/tmp/startMysql.sh'],
+ path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ notify => Anchor['hdp2-mysql::server::end'],
+ logoutput => "true"
+ }
+
+ anchor { 'hdp2-mysql::server::end':}
+
+ } else {
+ hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+ }
+}
Added: incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-nagios/files/check_aggregate.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-nagios/files/check_aggregate.php?rev=1398196&view=auto
==============================================================================
--- incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-nagios/files/check_aggregate.php (added)
+++ incubator/ambari/branches/branch-0.9-h2-dev/hmc/puppet/modules/hdp2-nagios/files/check_aggregate.php Mon Oct 15 05:55:46 2012
@@ -0,0 +1,247 @@
+<?php
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ $options = getopt ("f:s:n:w:c:t:");
+ if (!array_key_exists('t', $options) || !array_key_exists('f', $options) || !array_key_exists('w', $options)
+ || !array_key_exists('c', $options) || !array_key_exists('s', $options)) {
+ usage();
+ exit(3);
+ }
+ $status_file=$options['f'];
+ $status_code=$options['s'];
+ $type=$options['t'];
+ $warn=$options['w']; $warn = preg_replace('/%$/', '', $warn);
+ $crit=$options['c']; $crit = preg_replace('/%$/', '', $crit);
+ if ($type == "service" && !array_key_exists('n', $options)) {
+ echo "Service description not provided -n option\n";
+ exit(3);
+ }
+ if ($type == "service") {
+ $service_name=$options['n'];
+ /* echo "DESC: " . $service_name . "\n"; */
+ }
+
+ $result = array();
+ $status_file_content = file_get_contents($status_file);
+
+ $counts;
+ if ($type == "service") {
+ $counts=query_alert_count($status_file_content, $service_name, $status_code);
+ } else {
+ $counts=query_host_count($status_file_content, $status_code);
+ }
+
+ if ($counts['total'] == 0) {
+ $percent = 0;
+ } else {
+ $percent = ($counts['actual']/$counts['total'])*100;
+ }
+ if ($percent >= $crit) {
+ echo "CRITICAL: total:<" . $counts['total'] . ">, affected:<" . $counts['actual'] . ">\n";
+ exit (2);
+ }
+ if ($percent >= $warn) {
+ echo "WARNING: total:<" . $counts['total'] . ">, affected:<" . $counts['actual'] . ">\n";
+ exit (1);
+ }
+ echo "OK: total:<" . $counts['total'] . ">, affected:<" . $counts['actual'] . ">\n";
+ exit(0);
+
+
+ # Functions
+ /* print usage */
+ function usage () {
+ echo "Usage: $0 -f <status_file_path> -t type(host/service) -s <status_codes> -n <service description> -w <warn%> -c <crit%>\n";
+ }
+
+ /* Query host count */
+ function query_host_count ($status_file_content, $status_code) {
+ $num_matches = preg_match_all("/hoststatus \{([\S\s]*?)\}/", $status_file_content, $matches, PREG_PATTERN_ORDER);
+ $hostcounts_object = array ();
+ $total_hosts = 0;
+ $hosts = 0;
+ foreach ($matches[0] as $object) {
+ $total_hosts++;
+ if (getParameter($object, "current_state") == $status_code) {
+ $hosts++;
+ }
+ }
+ $hostcounts_object['total'] = $total_hosts;
+ $hostcounts_object['actual'] = $hosts;
+ return $hostcounts_object;
+ }
+
+ /* Query Alert counts */
+ function query_alert_count ($status_file_content, $service_name, $status_code) {
+ $num_matches = preg_match_all("/servicestatus \{([\S\s]*?)\}/", $status_file_content, $matches, PREG_PATTERN_ORDER);
+ $alertcounts_objects = array ();
+ $total_alerts=0;
+ $alerts=0;
+ foreach ($matches[0] as $object) {
+ if (getParameter($object, "service_description") == $service_name) {
+ $total_alerts++;
+ if (getParameter($object, "current_state") >= $status_code) {
+ $alerts++;
+ }
+ }
+ }
+ $alertcounts_objects['total'] = $total_alerts;
+ $alertcounts_objects['actual'] = $alerts;
+ return $alertcounts_objects;
+ }
+
+ function get_service_type($service_description)
+ {
+ $pieces = explode("::", $service_description);
+ switch ($pieces[0]) {
+ case "NAMENODE":
+ $pieces[0] = "HDFS";
+ break;
+ case "JOBTRACKER":
+ $pieces[0] = "MAPREDUCE";
+ break;
+ case "RESOURCEMANAGER":
+ $pieces[0] = "YARN";
+ break;
+ case "HBASEMASTER":
+ $pieces[0] = "HBASE";
+ break;
+ case "SYSTEM":
+ case "HDFS":
+ case "YARN":
+ case "MAPREDUCE":
+ case "HBASE":
+ break;
+ default:
+ $pieces[0] = "UNKNOWN";
+ }
+ return $pieces[0];
+ }
+
+ function getParameter($object, $key)
+ {
+ $pattern="/\s" . $key . "[\s= ]*([\S, ]*)\n/";
+ $num_mat = preg_match($pattern, $object, $matches);
+ $value = "";
+ if ($num_mat) {
+ $value = $matches[1];
+ }
+ return $value;
+ }
+
+function indent($json) {
+
+ $result = '';
+ $pos = 0;
+ $strLen = strlen($json);
+ $indentStr = ' ';
+ $newLine = "\n";
+ $prevChar = '';
+ $outOfQuotes = true;
+
+ for ($i=0; $i<=$strLen; $i++) {
+
+ // Grab the next character in the string.
+ $char = substr($json, $i, 1);
+
+ // Are we inside a quoted string?
+ if ($char == '"' && $prevChar != '\\') {
+ $outOfQuotes = !$outOfQuotes;
+
+ // If this character is the end of an element,
+ // output a new line and indent the next line.
+ } else if(($char == '}' || $char == ']') && $outOfQuotes) {
+ $result .= $newLine;
+ $pos --;
+ for ($j=0; $j<$pos; $j++) {
+ $result .= $indentStr;
+ }
+ }
+
+ // Add the character to the result string.
+ $result .= $char;
+
+ // If the last character was the beginning of an element,
+ // output a new line and indent the next line.
+ if (($char == ',' || $char == '{' || $char == '[') && $outOfQuotes) {
+ $result .= $newLine;
+ if ($char == '{' || $char == '[') {
+ $pos ++;
+ }
+
+ for ($j = 0; $j < $pos; $j++) {
+ $result .= $indentStr;
+ }
+ }
+
+ $prevChar = $char;
+ }
+
+ return $result;
+}
+
+/* JSON documment format */
+/*
+{
+ "programstatus":{
+ "last_command_check":"1327385743"
+ },
+ "hostcounts":{
+ "up_nodes":"",
+ "down_nodes":""
+ },
+ "hoststatus":[
+ {
+ "host_name"="ip-10-242-191-48.ec2.internal",
+ "current_state":"0",
+ "last_hard_state":"0",
+ "plugin_output":"PING OK - Packet loss = 0%, RTA = 0.04 ms",
+ "last_check":"1327385564",
+ "current_attempt":"1",
+ "last_hard_state_change":"1327362079",
+ "last_time_up":"1327385574",
+ "last_time_down":"0",
+ "last_time_unreachable":"0",
+ "is_flapping":"0",
+ "last_check":"1327385574",
+ "servicestatus":[
+ ]
+ }
+ ],
+ "servicestatus":[
+ {
+ "service_type":"HDFS", {HBASE, YARN, MAPREDUCE, HIVE, ZOOKEEPER}
+ "service_description":"HDFS Current Load",
+ "host_name"="ip-10-242-191-48.ec2.internal",
+ "current_attempt":"1",
+ "current_state":"0",
+ "plugin_output":"PING OK - Packet loss = 0%, RTA = 0.04 ms",
+ "last_hard_state_change":"1327362079",
+ "last_time_ok":"1327385479",
+ "last_time_warning":"0",
+ "last_time_unknown":"0",
+ "last_time_critical":"0",
+ "last_check":"1327385574",
+ "is_flapping":"0"
+ }
+ ]
+}
+*/
+
+?>