You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2011/06/02 21:13:46 UTC
svn commit: r1130814 - in /incubator/hcatalog/branches/branch-0.1:
CHANGES.txt build.xml scripts/ scripts/hcat_server_install.sh
src/docs/src/documentation/content/xdocs/install.xml
src/docs/src/documentation/content/xdocs/site.xml
Author: gates
Date: Thu Jun 2 21:13:45 2011
New Revision: 1130814
URL: http://svn.apache.org/viewvc?rev=1130814&view=rev
Log:
HCATALOG-21: Install documenation and script
Added:
incubator/hcatalog/branches/branch-0.1/scripts/
incubator/hcatalog/branches/branch-0.1/scripts/hcat_server_install.sh
incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/install.xml
Modified:
incubator/hcatalog/branches/branch-0.1/CHANGES.txt
incubator/hcatalog/branches/branch-0.1/build.xml
incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/site.xml
Modified: incubator/hcatalog/branches/branch-0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.1/CHANGES.txt?rev=1130814&r1=1130813&r2=1130814&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.1/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.1/CHANGES.txt Thu Jun 2 21:13:45 2011
@@ -44,6 +44,8 @@ Release 0.1 - Unreleased
IMPROVEMENTS
+ HCAT-21. Install documenation and script (gates)
+
HCAT-20. Create a package target (gates via hashutosh)
HCAT-29. Remove Yahoo copyright from documentation (Jakob Homan via hashutosh)
Modified: incubator/hcatalog/branches/branch-0.1/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.1/build.xml?rev=1130814&r1=1130813&r2=1130814&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.1/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.1/build.xml Thu Jun 2 21:13:45 2011
@@ -430,8 +430,11 @@
<fileset dir="conf" />
<fileset file="hive/external/conf/hive-default.xml"/>
</copy>
- <copy todir="${dist.dir}/scripts"
- file="hive/external/metastore/scripts/upgrade/mysql/hive-schema-0.7.0.mysql.sql"/>
+
+ <copy todir="${dist.dir}/scripts">
+ <fileset file="hive/external/metastore/scripts/upgrade/mysql/hive-schema-0.7.0.mysql.sql"/>
+ <fileset file="scripts/hcat_server_install.sh"/>
+ </copy>
<copy todir="${dist.dir}/src" includeEmptyDirs="true">
<fileset dir="${src.dir}" />
@@ -472,6 +475,9 @@
<tarfileset dir="${build.dir}" mode="755">
<include name="${final.name}/bin/*" />
</tarfileset>
+ <tarfileset dir="${build.dir}" mode="755">
+ <include name="${final.name}/scripts/*" />
+ </tarfileset>
</tar>
</target>
Added: incubator/hcatalog/branches/branch-0.1/scripts/hcat_server_install.sh
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.1/scripts/hcat_server_install.sh?rev=1130814&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.1/scripts/hcat_server_install.sh (added)
+++ incubator/hcatalog/branches/branch-0.1/scripts/hcat_server_install.sh Thu Jun 2 21:13:45 2011
@@ -0,0 +1,134 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script assumes that it is being run from the top level directory of the
+# HCatalog distribution tarball
+
+function usage() {
+ echo "Usage: $0 -r root -d dbroot -h hadoop_home"
+ echo " root is the directory where you like to install the HCatalog server"
+ echo " /usr/local/hcat is suggested."
+ echo " dbroot is the directory where your mysql connector jar is located."
+ echo " hadoop_home is the directory of your Hadoop installation."
+ echo " All paths must be absolute"
+}
+
+dir_check=`head -1 NOTICE.txt`
+if [ "${dir_check}" != "Apache HCatalog" ] ; then
+ echo "This script must be run in the top level directory of your HCatalog" \
+ "distribution."
+ exit 1
+fi
+
+root="unknown"
+dbroot="unknown"
+hadoop_home="unknown"
+
+while [ "${1}x" != "x" ] ; do
+ if [ $1 == "-r" ] || [ $1 == "--root" ] ; then
+ shift
+ root=$1
+ shift
+ elif [ $1 == "-d" ] || [ $1 == "--dbroot" ] ; then
+ shift
+ dbroot=$1
+ shift
+ elif [ $1 == "-h" ] || [ $1 == "--hadoop" ] ; then
+ shift
+ hadoop_home=$1
+ shift
+ else
+ echo "Unknown option $1"
+ shift
+ fi
+
+done
+
+for var in $root $dbroot $hadoop_home ; do
+ if [ $var == "unknown" ] ; then
+ usage
+ exit 1
+ fi
+done
+
+# Make sure root and dbroot are absolute paths
+
+for var in $root $dbroot $hadoop_home ; do
+ if [ ${var:0:1} != "/" ] ; then
+ usage
+ exit 1
+ fi
+done
+
+# Make sure root is writable and has the necessary directories
+root_owner=`ls -ld $root | awk '{print $3}'`
+if [ $root_owner != `whoami` ] ; then
+ echo "You must run this as the user that will run HCatalog and that user" \
+ "must own the root directory."
+ exit 1
+fi
+
+root_perms=`ls -ld $root | awk '{print $1}'`
+if [ ${root_perms:0:4} != "drwx" ] ; then
+ echo "Your root directory must be readable, writable, and executable by" \
+ "its owner."
+ exit 1
+fi
+
+# Check that the required Mysql driver is in the dbroot
+mysql_jar=`ls ${dbroot}/mysql-connector-java-*.jar 2>/dev/null | grep mysql-connector-java`
+if [ "${mysql_jar}x" == "x" ] ; then
+ echo "The required jar file mysql-connector-java-version.jar is not in " \
+ "$dbroot or is not readable"
+ exit 1
+fi
+
+# Create the needed directories in root
+for dir in var conf var/log bin lib ; do
+ if [ ! -d $root/$dir ] ; then
+ mkdir $root/$dir
+ fi
+done
+
+# Move files into the appropriate directories
+for dir in conf lib ; do
+ for file in ./$dir/* ; do
+ cp $file $root/$dir
+ done
+done
+
+# Move the proto-hive-site.xml to hive-site.xml
+cp $root/conf/proto-hive-site.xml $root/conf/hive-site.xml
+
+# Set permissions on hive-site.xml to 700, since it will contain the password to the
+# database
+chmod 700 $root/conf/hive-site.xml
+
+# Write out an environment file so that the start file can use it later
+cat > $root/conf/hcat-env.sh <<!!
+ROOT=$root
+DBROOT=$dbroot
+USER=`whoami`
+HADOOP_HOME=$hadoop_home
+!!
+
+echo "Installation successful"
+
+
+
+
Added: incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/install.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/install.xml?rev=1130814&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/install.xml (added)
+++ incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/install.xml Thu Jun 2 21:13:45 2011
@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>Installing HCatalog</title>
+ </header>
+ <body>
+
+ <section>
+ <title>Server Installation</title>
+
+ <p><strong>Prerequisites</strong></p>
+ <ul>
+ <li>Machine on which the server can be installed - this should have
+ access to the hadoop cluster in question, and be accessible from
+ the machines you launch jobs from</li>
+ <li>MySQL db</li>
+ <li>Hadoop cluster</li>
+ <li>Unix user that the server will run as, and an associated kerberos
+ service principal and keytabs.</li>
+ </ul>
+
+ <p>Throughout these instructions when you see a word in <em>italics</em> it
+ indicates a place where you should replace the word with a locally
+ appropriate value such as a hostname or password.</p>
+
+ <p><strong>Database Setup</strong></p>
+
+ <p>Select a machine to install the database on. This need not be the same
+ machine as the Thrift server, which we will set up later. For large
+ clusters we recommend that they not be the same machine. For the
+ purposes of these instructions we will refer to this machine as
+ <em>hcatdb.acme.com</em></p>
+
+ <p>Install MySQL server on <em>hcatdb.acme.com</em>. You can obtain
+ packages for MySQL from <a href="http://www.mysql.com/downloads/">MySQL's
+ download site</a>. We have developed and tested with versions 5.1.46
+ and 5.1.48. We suggest you use these versions or later.
+ Once you have MySQL up and running, use the <code>mysql</code> command line
+ tool to add the <code>hive</code> user and <code>hivemetastoredb</code>
+ database. You will need to pick a password for your <code>hive</code>
+ user, and replace <em>dbpassword</em> in the following commands with it.</p>
+
+ <p><code>mysql -u root</code></p>
+ <p><code>mysql> CREATE USER 'hive'@'</code><em>hcatdb.acme.com</em><code>' IDENTIFIED BY '</code><em>dbpassword</em><code>';</code></p>
+ <p><code>mysql> CREATE DATABASE hivemetastoredb DEFAULT CHARACTER SET latin1 DEFAULT COLLATE latin1_swedish_ci;</code></p>
+ <p><code>mysql> GRANT ALL PRIVILEGES ON hivemetastoredb.* TO 'hive'@'</code><em>hcatdb.acme.com</em><code>' WITH GRANT OPTION;</code></p>
+ <p><code>mysql> flush privileges;</code></p>
+ <p><code>mysql> quit;</code></p>
+
+ <p>In a temporary directory, untar the HCatalog artifact</p>
+
+ <p><code>tar xzf hcatalog-</code><em>version</em><code>.tar.gz</code></p>
+
+ <p>Use the database installation script found in the package to create the
+ database</p>
+ <p><code>mysql -u hive -D hivemetastoredb -h</code><em>hcatdb.acme.com</em><code> -p < scripts/hive-schema-0.7.0.mysql.sql</code></p>
+
+ <p><strong>Thrift Server Setup</strong></p>
+
+ <p>Select a machine to install your Thrift server on. For smaller and test
+ installations this can be the same machine as the database. For the
+ purposes of these instructions we will refer to this machine as
+ <em>hcatsvr.acme.com</em>.</p>
+
+ <p>Install the MySQL Java connector libraries on <em>hcatsvr.acme.com</em>.
+ You can obtain these from
+ <a href="http://www.mysql.com/downloads/connector/j/5.1.html">MySQL's
+ download site</a>.</p>
+
+ <p>Select a user to run the Thrift server as. This user should not be a
+ human user, and must be able to act as a proxy for other users. We suggest
+ the name "hcat" for the user. Throughout the rest of this documentation
+ we will refer to this user as "hcat". If necessary, add the user to
+ <em>hcatsvr.acme.com</em>.</p>
+
+ <p>Select a <em>root</em> directory for your installation of HCatalog. This
+ directory must be owned by the hcat user. We recommend
+ <code>/usr/local/hcat</code>. If necessary, create the directory.</p>
+
+ <p>Download the HCatalog release into a temporary directory, and untar
+ it. Then change directories into the new distribution and run the HCatalog
+ server installation script. You will need to know the directory you chose
+ as <em>root</em> and the
+ directory you installed the MySQL Java connector libraries into (referred
+ to in the command below as <em>dbroot</em>).</p>
+
+ <p><code>tar zxf hcatalog-</code><em>version</em><code>.tar.gz
+ cd hcatalog-</code><em>version</em></p>
+ <p><code>scripts/hcat_server_install.sh -r </code><em>root</em><code> -d </code><em>dbroot</em></p>
+
+ <p>Now you need to edit your <em>root</em><code>/conf/hive-site.xml</code> file.
+ Open this file in your favorite text editor. The following table shows the
+ values you need to configure.</p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value to Set it to</th>
+ </tr>
+ <tr>
+ <td>javax.jdo.option.ConnectionURL</td>
+ <td>In the JDBC connection string, change DBHOSTNAME to the name
+ of the machine you put the MySQL server on.</td>
+ </tr>
+ <tr>
+ <td>javax.jdo.option.ConnectionPassword</td>
+ <td><em>dbpassword</em> value you used in setting up the MySQL server
+ above</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.warehouse.dir</td>
+ <td>The directory you want to use for the default database in your
+ installation</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.uris</td>
+ <td>You need to set the hostname to your Thrift
+ server. Replace <em>SVRHOST</em> with the name of the
+ machine you are installing the Thrift server on. You can also
+ change the port the Thrift server runs on by changing the default
+ value of 3306.</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.sasl.enabled</td>
+ <td>Set to true by default. Set to false if you do not wish to
+ secure the thrift interface. This can be convenient for testing.
+ We do not recommend turning this off in production.</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.kerberos.keytab.file</td>
+ <td>The path to the Kerberos keytab file containg the metastore
+ thrift server's service principal.</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.kerberos.principal</td>
+ <td>The service principal for the metastore thrift server. You can
+ reference your host as _HOST and it will be replaced with your
+ actual hostname</td>
+ </tr>
+ </table>
+
+ <p>You can now procede to starting the server.</p>
+ </section>
+
+ <section>
+ <title>Starting the Server</title>
+
+ <p>Start the HCatalog server by switching directories to
+ <em>root</em> and invoking the start script
+ <code>bin/hcat_server_start.sh</code></p>
+
+ </section>
+
+ <section>
+ <title>Logging</title>
+
+ <p>Server activity logs and gc logs are located in
+ <em>root</em><code>/var/log/hcat_server</code>. Logging configuration is located at
+ <em>root</em><code>/conf/log4j.properties</code>. Server logging uses
+ <code>DailyRollingFileAppender</code> by default. It will generate a new
+ file per day and does not expire old log files automatically.</p>
+
+ </section>
+
+ <section>
+ <title>Stopping the Server</title>
+
+ <p>To stop the HCatalog server, change directories to the <em>root</em>
+ directory and invoke the stop script
+ <code>bin/hcat_server_stop.sh</code></p>
+
+ </section>
+
+ <section>
+ <title>Client Install</title>
+
+ <p>Select a <em>root</em> directory for your installation of HCatalog client.
+ We recommend <code>/usr/local/hcat</code>. If necessary, create the directory.</p>
+
+ <p>Download the HCatalog release into a temporary directory, and untar
+ it.</p>
+
+ <p><code>tar zxf hcatalog-</code><em>version</em><code>.tar.gz</code></p>
+
+ <p>Now you need to edit your <em>root</em><code>/conf/hive-site.xml</code> file.
+ Open this file in your favorite text editor. The following table shows the
+ values you need to configure. These values should match the values set on
+ the HCatalog server. Do <strong>NOT</strong> copy the configuration file
+ from your server installation as that contains the password to your
+ database, which you should not distribute to your clients.</p>
+
+ <table>
+ <tr>
+ <th>Parameter</th>
+ <th>Value to Set it to</th>
+ </tr>
+ <tr>
+ <td>hive.metastore.warehouse.dir</td>
+ <td>The directory you want to use for the default database in your
+ installation</td>
+ </tr>
+ <tr>
+ <td>hive.metastore.uris</td>
+ <td>You need to set the hostname wish your Thrift
+ server to use by replacing <em>SVRHOST</em> with the name of the
+ machine you are installing the Thrift server on. You can also
+ change the port the Thrift server runs on by changing the default
+ value of 3306.</td>
+ </tr>
+ </table>
+
+ <p>The HCatalog command line interface (CLI) can now be invoked as
+ <em>root</em><code>/bin/hcat.sh</code>.</p>
+
+ </section>
+
+ </body>
+</document>
Modified: incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/site.xml?rev=1130814&r1=1130813&r2=1130814&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ incubator/hcatalog/branches/branch-0.1/src/docs/src/documentation/content/xdocs/site.xml Thu Jun 2 21:13:45 2011
@@ -44,6 +44,7 @@ See http://forrest.apache.org/docs/linki
<index label="MapReduce Input & Output " href="inputoutput.html" />
<index label="Cmd Line Interface " href="cli.html" />
<index label="Supported data formats" href="supportedformats.html" />
+ <index label="Installation" href="install.html" />
</docs>
</site>