You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/07/25 20:29:49 UTC
svn commit: r1365722 [3/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/
hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/
src/docs/src/documentation/content/xdocs/images/
src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet...
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>GET queue </title>
+ </header>
+
+ <body>
+ <section>
+ <title>Description</title>
+ <p>Return a list of all job IDs registered to the user.</p>
+ </section>
+
+ <section>
+ <title>URL</title>
+ <p><code>http://</code>www.myserver.com<code>/templeton/v1/queue</code></p>
+ </section>
+
+ <section>
+ <title>Parameters</title>
+ <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+ are accepted.</p>
+ </section>
+
+ <section>
+ <title>Results</title>
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+ <tr><td><strong>ids</strong></td>
+ <td>A list of all job IDs registered to the user.</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Example</title>
+
+ <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/queue?user.name=ctdean'
+</source>
+
+ <p><strong>JSON Output</strong></p>
+<source>
+{
+ "job_201111111311_0008",
+ "job_201111111311_0012"
+}
+</source>
+ </section>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>POST ddl/database/:db/table/:table</title>
+ </header>
+
+ <body>
+ <section>
+ <title>Description</title>
+ <p>Rename an HCatalog table.</p>
+ </section>
+
+ <section>
+ <title>URL</title>
+ <p><code>http://</code>www.myserver.com<code>/templeton/v1/ddl/database/</code>:db<code>/table/</code>:table</p>
+ </section>
+
+ <section>
+ <title>Parameters</title>
+ <table>
+ <tr><th>Name</th><th>Description</th><th>Required?</th><th>Default</th></tr>
+
+ <tr><td><strong>:db</strong></td>
+ <td>The database name</td>
+ <td>Required</td>
+ <td>None</td>
+ </tr>
+
+ <tr><td><strong>:table</strong></td>
+ <td>The existing (old) table name</td>
+ <td>Required</td>
+ <td>None</td>
+ </tr>
+
+ <tr><td><strong>rename</strong></td>
+ <td>The new table name</td>
+ <td>Required</td>
+ <td>None</td>
+ </tr>
+
+ <tr><td><strong>group</strong></td>
+ <td>The user group to use</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
+ <tr><td><strong>permissions</strong></td>
+ <td>The permissions string to use. The format is
+ "<code>rwxrw-r-x</code>".</td>
+ <td>Optional</td>
+ <td>None</td>
+ </tr>
+
+ </table>
+ </section>
+
+ <section>
+ <title>Results</title>
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+
+ <tr><td><strong>table</strong></td>
+ <td>The new table name</td>
+ </tr>
+
+ <tr><td><strong>database</strong></td>
+ <td>The database name</td>
+ </tr>
+
+ </table>
+ </section>
+
+ <section>
+ <title>Example</title>
+
+ <p><strong>Curl Command</strong></p>
+<source>
+% curl -s -d user.name=ctdean \
+ -d rename=test_table_2 \
+ 'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table'
+</source>
+
+ <p><strong>JSON Output</strong></p>
+<source>
+{
+ "table": "test_table_2",
+ "database": "default"
+}
+</source>
+
+ <p><strong>JSON Output (error)</strong></p>
+<source>
+{
+ "error": "Table test_table does not exist",
+ "errorCode": 404,
+ "database": "default",
+ "table": "test_table_2"
+}
+</source>
+ </section>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>Templeton Resources </title>
+ </header>
+ <body>
+ <table>
+ <tr><th>Resource</th><th>Description</th></tr>
+
+ <tr><td><a href="responsetypes.html">:version</a></td>
+ <td>Returns a list of supported response types.</td></tr>
+
+ <tr><td><a href="status.html">status</a></td>
+ <td>Returns the Templeton server status.</td></tr>
+
+ <tr><td><a href="versions.html">version</a></td>
+ <td>Returns the a list of supported versions and the current version.</td></tr>
+
+ <tr><td><a href="ddl.html">ddl</a></td>
+ <td>Performs an HCatalog DDL command.</td></tr>
+
+ <tr><td><a href="listdbs.html">ddl/database</a></td>
+ <td>List HCatalog databases.</td></tr>
+
+ <tr><td><a href="descdb.html">ddl/database/:db (GET)</a></td>
+ <td>Describe an HCatalog database.</td></tr>
+
+ <tr><td><a href="createdb.html">ddl/database/:db (PUT)</a></td>
+ <td>Create an HCatalog database.</td></tr>
+
+ <tr><td><a href="deletedb.html">ddl/database/:db (DELETE)</a></td>
+ <td>Delete (drop) an HCatalog database.</td></tr>
+
+ <tr><td><a href="listtables.html">ddl/database/:db/table</a></td>
+ <td>List the tables in an HCatalog database.</td></tr>
+
+ <tr><td><a href="desctable.html">ddl/database/:db/table/:table (GET)</a></td>
+ <td>Describe an HCatalog table.</td></tr>
+
+ <tr><td><a href="createtable.html">ddl/database/:db/table/:table (PUT)</a></td>
+ <td>Create a new HCatalog table.</td></tr>
+
+ <tr><td><a href="renametable.html">ddl/database/:db/table/:table (POST)</a></td>
+ <td>Rename an HCatalog table.</td></tr>
+
+ <tr><td><a href="deletetable.html">ddl/database/:db/table/:table (DELETE)</a></td>
+ <td>Delete (drop) an HCatalog table.</td></tr>
+
+ <tr><td><a href="createtablelike.html">ddl/database/:db/table/:existingtable/like/:newtable (PUT)</a></td>
+ <td>Create a new HCatalog table like an existing one.</td></tr>
+
+ <tr><td><a href="listpartitions.html">ddl/database/:db/table/:table/partion</a></td>
+ <td>List all partitions in an HCatalog table.</td></tr>
+
+ <tr><td><a href="descpartition.html">ddl/database/:db/table/:table/partion/:partition (GET)</a></td>
+ <td>Describe a single partition in an HCatalog table.</td></tr>
+
+ <tr><td><a href="createpartition.html">ddl/database/:db/table/:table/partion/:partition (PUT)</a></td>
+ <td>Create a partition in an HCatalog table.</td></tr>
+
+ <tr><td><a href="deletepartition.html">ddl/database/:db/table/:table/partion/:partition (DELETE)</a></td>
+ <td>Delete (drop) a partition in an HCatalog table.</td></tr>
+
+ <tr><td><a href="listcolumns.html">ddl/database/:db/table/:table/column</a></td>
+ <td>List the columns in an HCatalog table.</td></tr>
+
+ <tr><td><a href="desccolumn.html">ddl/database/:db/table/:table/column/:column (GET)</a></td>
+ <td>Describe a single column in an HCatalog table.</td></tr>
+
+ <tr><td><a href="createcolumn.html">ddl/database/:db/table/:table/column/:column (PUT)</a></td>
+ <td>Create a column in an HCatalog table.</td></tr>
+
+ <tr><td><a href="listproperties.html">ddl/database/:db/table/:table/property (GET)</a></td>
+ <td>List table properties.</td></tr>
+
+ <tr><td><a href="descproperty.html">ddl/database/:db/table/:table/property/:property (GET)</a></td>
+ <td>Return the value of a single table property.</td></tr>
+
+ <tr><td><a href="addproperty.html">ddl/database/:db/table/:table/property/:property (PUT)</a></td>
+ <td>Set a table property.</td></tr>
+
+ <tr><td><a href="mapreducestreaming.html">mapreduce/streaming</a></td>
+ <td>Creates and queues Hadoop streaming MapReduce jobs.</td></tr>
+
+ <tr><td><a href="mapreducejar.html">mapreduce/jar</a></td>
+ <td>Creates and queues standard Hadoop MapReduce jobs.</td></tr>
+
+ <tr><td><a href="pig.html">pig</a></td>
+ <td>Creates and queues Pig jobs.</td></tr>
+
+ <tr><td><a href="hive.html">hive</a></td>
+ <td>Runs Hive queries and commands.</td></tr>
+
+ <tr><td><a href="queuelist.html">queue</a></td>
+ <td>Returns a list of all jobids registered for the user.</td></tr>
+
+ <tr><td><a href="queue.html">queue/:jobid (GET)</a></td>
+ <td>Returns the status of a job given its ID.</td></tr>
+
+ <tr><td><a href="queuedelete.html">queue/:jobid (DELETE)</a></td>
+ <td>Kill a job given its ID.</td></tr>
+
+ </table>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>GET :version </title>
+ </header>
+
+ <body>
+ <section>
+ <title>Description</title>
+ <p>Returns a list of the response types supported by Templeton.</p>
+ </section>
+
+ <section>
+ <title>URL</title>
+ <p><code>http://</code>www.myserver.com<code>/templeton/</code>:version</p>
+ </section>
+
+ <section>
+ <title>Parameters</title>
+ <table>
+ <tr><th>Name</th><th>Description</th><th>Required?</th><th>Default</th></tr>
+
+ <tr><td><strong>:version</strong></td>
+ <td>The Templeton version number. (Currently this must be "v1")</td>
+ <td>Required</td>
+ <td>None</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Results</title>
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+
+ <tr><td><strong>responseTypes</strong></td>
+ <td>A list of all supported response types</td>
+ </tr>
+
+ </table>
+ </section>
+
+ <section>
+ <title>Example</title>
+
+ <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1'
+</source>
+
+ <p><strong>JSON Output</strong></p>
+<source>
+{
+ "responseTypes": [
+ "application/json"
+ ]
+}
+</source>
+
+ <p><strong>JSON Output (error)</strong></p>
+<source>
+{
+ "error": "null for uri: http://localhost:50111/templeton/v2"
+}
+</source>
+ </section>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,140 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>HCatalog REST API </title>
+ </header>
+ <body>
+
+ <section>
+ <title>Introduction </title>
+ <p>This document describes HCatalog REST API.
+ As shown in the figure below, developers make HTTP requests to access
+ <a href="http://hadoop.apache.org/mapreduce/">Hadoop MapReduce</a>,
+ <a href="http://pig.apache.org/">Pig</a>,
+ <a href="http://hive.apache.org/">Hive</a>, and
+ <a href="http://incubator.apache.org/hcatalog/docs/r0.2.0/cli.html">
+ HCatalog DDL</a> from within applications.
+ Data and code used by this API is maintained in
+ <a href="http://hadoop.apache.org/hdfs/">HDFS</a>. HCatalog DDL commands
+ are executed directly when requested.
+ MapReduce, Pig, and Hive jobs are placed in queue by
+ and can be monitored for progress or stopped as required.
+ Developers specify a location
+ in HDFS into which Pig, Hive, and MapReduce results should be placed.</p>
+ <figure src="images/TempletonArch.jpg" align="left" alt="Templeton Architecture"/>
+ </section>
+
+ <section>
+ <title>URL format </title>
+ <p>HCatalog's REST resources are accessed using the following URL format:</p>
+ <p><code>http://</code><em>yourserver</em><code>/templeton/v1/</code><em>resource</em></p>
+ <p>where "<em>yourserver</em>" is replaced with your server name, and
+ "<em>resource</em>" is replaced by the HCatalog
+ resource name.</p>
+ <p>For example, to check if the server is running you could
+ access the following URL:</p>
+ <p><code>http://www.myserver.com/templeton/v1/status</code></p>
+ </section>
+
+ <section>
+ <title>Security </title>
+ <p>The current version supports two types of security:</p>
+ <ul>
+ <li>Default security (without additional authentication)</li>
+ <li>Authentication via
+ <a href="http://web.mit.edu/kerberos/">Kerberos</a></li>
+ </ul>
+ <section>
+ <title>Standard Parameters </title>
+ <p>Every REST resource can accept the following parameters to
+ aid in authentication: </p>
+ <ul>
+ <li>user.name: The user name as a string.
+ Only valid when using default security. </li>
+ <li>SPNEGO credentials: When running with Kerberos authentication. </li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Security Error Response</title>
+ <p>If the user.name parameter is not supplied when required,
+ the following error will be returned: </p>
+<source>
+{
+ "error": "No user found. Missing user.name parameter."
+}
+</source>
+ </section>
+
+ </section>
+
+ <section>
+ <title>WebHDFS and Code Push</title>
+ <p>Data and code that are used by HCatalog's REST resources must first be placed in
+ Hadoop. When placing files into HDFS is required you can use
+ whatever method is most convienient for you. We suggest WebHDFS since it provides
+ a REST interface for moving files into and out of HDFS.</p>
+ </section>
+
+ <section>
+ <title>Error Codes and Responses</title>
+ <p>The server returns the following HTTP status codes.</p>
+ <ul>
+ <li><strong>200 OK:</strong> Success!</li>
+ <li><strong>400 Bad Request:</strong> The request was invalid.</li>
+ <li><strong>401 Unauthorized:</strong> Credentials were missing or incorrect.</li>
+ <li><strong>404 Not Found:</strong> The URI requested is invalid or the
+ resource requested does not exist.</li>
+ <li><strong>500 Internal Server Error:</strong> We received an unexpected result.</li>
+ <li><strong>503 Busy, please retry:</strong> The server is busy.</li>
+ </ul>
+ <p>Other data returned directly by the server is returned in JSON format.
+ JSON responses are limited to 1MB in size. Responses over this limit must be
+ stored into HDFS using provided options instead of being directly returned.
+ If an HCatalog DDL command might return results greater than 1MB, it's
+ suggested that a corresponding Hive request be executed instead.</p>
+ </section>
+
+ <section>
+ <title>Log Files</title>
+ <p>The server creates three log files when in operation:</p>
+ <ul>
+ <li><strong>templeton.log</strong> is the log4j log. This the main log the application
+ writes to.</li>
+ <li><strong>templeton-console.log</strong> is what Java writes to stdout when the server is
+ started. It is a small amount of data, similar to "hcat.out".</li>
+ <li><strong>tempelton-console-error.log</strong> is what Java writes to stderr, similar to
+ "hcat.err".</li>
+ </ul>
+ <p>In the tempelton-log4j.properties file you can set the location of these logs using the
+ variable templeton.log.dir. This log4j.properties file is set in the server startup script.</p>
+ </section>
+
+ <section>
+ <title>Project Name</title>
+ <p>The original work to add REST APIs to HCatalog was called Templeton. For backward compatibility
+ the name still appears in URLs, log file names, etc. The Templeton name is taken from a character
+ in the award-winning children's novel Charlotte's Web, by E. B. White. The novel's protagonist is a pig named
+ Wilber. Templeton is a rat who helps Wilber by running errands and making deliveries as
+ requested by Charlotte while spinning her web.</p>
+ </section>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>Installation </title>
+ </header>
+ <body>
+
+ <section>
+ <title>Procedure</title>
+ <ol>
+ <li>Ensure that the
+ <a href="rest_server_install.html#Requirements">required related installations</a>
+ are in place, and place required files into the
+ <a href="rest_server_install.html#Hadoop+Distributed+Cache">Hadoop distributed cache.</a></li>
+ <li>Download and unpack the HCatalog distribution.</li>
+ <li>Set the <code>TEMPLETON_HOME</code> environment variable to the base of the
+ HCatalog REST server installation. This will usually be <code>HCATALOG_HOME/webhcat</code>.
+ This is used to find the Templeton
+ configuration.</li>
+ <li>Review the <a href="configuration.html">configuration</a>
+ and update or create <code>templeton-site.xml</code> as required. Ensure that
+ site specific component installation locations are accurate, especially
+ the Hadoop configuration path. Configuration variables that use a filesystem
+ path try to have reasonable defaults, but it's always safe to specify a full
+ and complete path.</li>
+ <li>Verify that HCatalog is installed and that the <code>hcat</code>
+ executable is in the PATH.</li>
+ <li>Build HCatalog using the command <code>ant jar</code> from the
+ top level HCatalog directory.</li>
+ <li>Start the REST server with the command
+ <code>bin/templeton_server.sh start</code>.</li>
+ <li>Check that your local install works. Assuming that the server is running
+ on port 8080, the following command would give output similar to that shown.
+<source>
+% curl -i http://localhost:50111/templeton/v1/status
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(7.6.0.v20120127)
+
+{"status":"ok","version":"v1"}
+%
+</source></li>
+ </ol>
+ </section>
+
+ <section>
+ <title>Server Commands</title>
+ <ul>
+ <li><strong>Start the server:</strong> <code>bin/templeton_server.sh start</code></li>
+ <li><strong>Stop the server:</strong> <code>bin/templeton_server.sh stop</code></li>
+ <li><strong>End-to-end build, run, test:</strong> <code>ant e2e</code></li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Requirements</title>
+ <ul>
+ <li><a href="http://ant.apache.org/">Ant</a>, version 1.8 or higher</li>
+ <li><a href="http://hadoop.apache.org/">Hadoop</a>, version 0.20.205.0</li>
+ <li><a href="http://zookeeper.apache.org/">ZooKeeper</a> is required if you are using
+ the ZooKeeper storage class. (Be sure to review
+ and update the ZooKeeper related <a href="configuration.html">Templeton
+ configuration</a>.)</li>
+ <li>
+ <a href="http://incubator.apache.org/hcatalog/">HCatalog</a>.
+ Version 0.4.1 or higher. The hcat executable must be both in the
+ PATH and properly configured in the <a
+ href="configuration.html">Templeton configuration</a>.
+ </li>
+ <li>
+ Permissions must be given to the user running the server.
+ (see below)
+ </li>
+ <li>
+ If running a secure cluster, Kerberos keys and principals must be
+ created. (see below)
+ </li>
+ <li><a href="rest_server_install.html#Hadoop+Distributed+Cache">Hadoop Distributed Cache</a>.
+ To use the <a href="http://hive.apache.org/">Hive</a>,
+ <a href="http://pig.apache.org/">Pig</a>, or
+ <a href="http://hadoop.apache.org/common/docs/r1.0.0/streaming.html">hadoop/streaming</a>
+ resources, see instructions below for placing the required files in the
+ Hadoop Distributed Cache.</li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Hadoop Distributed Cache</title>
+
+ <p>The server requires some files be accessible on the
+ <a href="http://hadoop.apache.org/common/docs/current/mapred_tutorial.html#DistributedCache">
+ Hadoop distributed cache</a>. For example, to avoid the installation of Pig and Hive
+ everywhere on the cluster, the server gathers a version of Pig or Hive from the
+ Hadoop distributed cache whenever those resources are invoked. After placing the
+ following components into HDFS please update the site configuration as required for
+ each.</p>
+
+ <ul>
+ <li><strong>Hive</strong>:
+ <a href="http://www.apache.org/dyn/closer.cgi/incubator/hcatalog/">Download</a>
+ the HCatalog tar.gz file and place it in HDFS. (If you need a version
+ that is not yet released, you may need to build it yourself following the HCatalog
+ instructions.)
+<source>
+hadoop fs -put /tmp/hcatalog-0.3.0.tar.gz /user/templeton/hcatalog-0.3.0.tar.gz
+</source>
+ </li>
+
+ <li><strong>Pig</strong>:
+ <a href="http://www.apache.org/dyn/closer.cgi/pig">Download</a>
+ the Pig tar.gz file and place it into HDFS. For example:
+<source>
+hadoop fs -put /tmp/pig-0.9.2.tar.gz /user/templeton/pig-0.9.2.tar.gz
+</source>
+ </li>
+
+ <li><strong>Hadoop Streaming</strong>:
+ Place <code>hadoop-streaming.jar</code> into HDFS. For example, use the
+ following command, substituting your path to the jar for the one below.
+<source>
+hadoop fs -put $HADOOP_PREFIX/hadoop-0.20.205.0/contrib/streaming/hadoop-streaming-0.20.205.0.jar \
+ /user/templeton/hadoop-streaming.jar
+</source>
+ </li>
+
+ <li><strong>Override Jars</strong>:
+ Place override jars required (if any) into HDFS. <em>Note</em>: As of this writing, all
+ released versions of Hadoop require a patch to properly run Templeton. This patch is
+ distributed with Templeton (located at
+ <code>templeton/src/hadoop_temp_fix/ugi.jar</code>)
+ and should be placed into HDFS, as reflected in the current default configuration.
+<source>
+hadoop fs -put ugi.jar /user/templeton/ugi.jar
+</source>
+ </li>
+ </ul>
+
+ <p>The location of these files in the cache, and the location
+ of the installations inside the archives, can be specified using the following
+ Templeton configuration variables. (See the
+ <a href="configuration.html">Configuration</a> documentation for more information
+ on changing Templeton configuration parameters.)</p>
+
+ <table>
+ <tr><th>Name</th><th>Default</th><th>Description</th></tr>
+
+ <tr>
+ <td><strong>templeton.pig.archive</strong></td>
+ <td><code>hdfs:///user/templeton/pig-0.9.2.tar.gz</code></td>
+ <td>The path to the Pig archive.</td>
+ </tr>
+
+ <tr>
+ <td><strong>templeton.pig.path</strong></td>
+ <td><code>pig-0.9.2.tar.gz/pig-0.9.2/bin/pig</code></td>
+ <td>The path to the Pig executable.</td>
+ </tr>
+
+ <tr>
+ <td><strong>templeton.hive.archive</strong></td>
+ <td><code>hdfs:///user/templeton/hcatalog-0.3.0.tar.gz</code></td>
+ <td>The path to the Hive archive.</td>
+ </tr>
+
+ <tr>
+ <td><strong>templeton.hive.path</strong></td>
+ <td><code>hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive</code></td>
+ <td>The path to the Hive executable.</td>
+ </tr>
+
+ <tr>
+ <td><strong>templeton.streaming.jar</strong></td>
+ <td><code>hdfs:///user/templeton/hadoop-streaming.jar</code></td>
+ <td>The path to the Hadoop streaming jar file.</td>
+ </tr>
+
+ <tr>
+ <td><strong>templeton.override.jars</strong></td>
+ <td><code>hdfs:///user/templeton/ugi.jar</code></td>
+ <td>Jars to add to the HADOOP_CLASSPATH for all Map Reduce jobs.
+ These jars must exist on HDFS.</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Permissions</title>
+ <p>
+ Permission must given for the user running the templeton
+ executable to run jobs for other users. That is, the templeton
+ server will impersonate users on the Hadoop cluster.
+ </p>
+
+ <p>
+ Create (or assign) a Unix user who will run the Templeton server.
+ Call this USER. See the Secure Cluster section below for choosing
+ a user on a Kerberos cluster.
+ </p>
+
+ <p>
+ Modify the Hadoop core-site.xml file and set these properties:
+ </p>
+
+ <table>
+ <tr><th>Variable</th><th>Value</th></tr>
+ <tr>
+ <td>hadoop.proxyuser.USER.groups</td>
+ <td>
+ A comma separated list of the Unix groups whose users will be
+ impersonated.
+ </td>
+ </tr>
+ <tr>
+ <td>hadoop.proxyuser.USER.hosts</td>
+ <td>
+ A comma separated list of the hosts that will run the hcat and
+ JobTracker servers.
+ </td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Secure Cluster</title>
+ <p>
+ To run Templeton on a secure cluster follow the Permissions
+ instructions above but create a Kerberos principal for the
+ Templeton server with the name <code>USER/host@realm</code>
+ </p>
+
+ <p>
+ Also, set the templeton configuration variables
+ <code>templeton.kerberos.principal</code> and
+ <code>templeton.kerberos.keytab</code>
+ </p>
+ </section>
+
+ </body>
+</document>
Modified: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml?rev=1365722&r1=1365721&r2=1365722&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml Wed Jul 25 20:29:44 2012
@@ -47,6 +47,48 @@ See http://forrest.apache.org/docs/linki
<index label="Storage Formats" href="supportedformats.html" />
<index label="Dynamic Partitioning" href="dynpartition.html" />
<index label="Notification" href="notification.html" />
+ <index label="WebHCat">
+ <index label="Overview" href="rest.html" />
+ <index label="Installation" href="rest_server_install.html" />
+ <index label="Configuration" href="configuration.html" />
+ <index label="Reference">
+ <index label="Resource List" href="resources.html" />
+ <index label=":version" href="responsetypes.html" />
+ <index label="status" href="status.html" />
+ <index label="version" href="versions.html" />
+ <index label="ddl">
+ <index label="ddl (summary)" href="ddlsummary.html" />
+ <index label="ddl" href="ddl.html" />
+ <index label="database" href="listdbs.html" />
+ <index label=":db (GET)" href="descdb.html" />
+ <index label=":db (PUT)" href="createdb.html" />
+ <index label=":db (DELETE)" href="deletedb.html" />
+ <index label="table" href="listtables.html" />
+ <index label=":table (GET)" href="desctable.html" />
+ <index label=":table (PUT)" href="createtable.html" />
+ <index label=":table (POST)" href="renametable.html" />
+ <index label=":table (DELETE)" href="deletetable.html" />
+ <index label="like/:table (PUT)" href="createtablelike.html" />
+ <index label="partition" href="listpartitions.html" />
+ <index label=":partition (GET)" href="descpartition.html" />
+ <index label=":partition (PUT)" href="createpartition.html" />
+ <index label=":partition (DELETE)" href="deletepartition.html" />
+ <index label="column" href="listcolumns.html" />
+ <index label=":column (GET)" href="desccolumn.html" />
+ <index label=":column (PUT)" href="createcolumn.html" />
+ <index label="property" href="listproperties.html" />
+ <index label=":property (GET)" href="descproperty.html" />
+ <index label=":property (PUT)" href="addproperty.html" />
+ </index>
+ <index label="mapreduce/streaming" href="mapreducestreaming.html" />
+ <index label="mapreduce/jar" href="mapreducejar.html" />
+ <index label="pig" href="pig.html" />
+ <index label="hive" href="hive.html" />
+ <index label="queue" href="queuelist.html" />
+ <index label=":jobid (GET)" href="queue.html" />
+ <index label=":jobid (DELETE)" href="queuedelete.html" />
+ </index>
+ </index>
<api label="API Docs" href="api/index.html"/>
</docs>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>GET status </title>
+ </header>
+
+ <body>
+ <section>
+ <title>Description</title>
+ <p>Returns the current status of the Templeton server.
+ Useful for heartbeat monitoring.</p>
+ </section>
+
+ <section>
+ <title>URL</title>
+ <p><code>http://</code>www.myserver.com<code>/templeton/v1/status</code></p>
+ </section>
+
+ <section>
+ <title>Parameters</title>
+ <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+ are accepted.</p>
+ </section>
+
+ <section>
+ <title>Results</title>
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+
+ <tr><td><strong>status</strong></td>
+ <td>"ok" if the Templeton server was contacted.</td>
+ </tr>
+
+ <tr><td><strong>version</strong></td>
+ <td>String containing the version number similar to "v1".</td>
+ </tr>
+ </table>
+ </section>
+
+ <section>
+ <title>Example</title>
+
+ <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/status'
+</source>
+
+ <p><strong>JSON Output</strong></p>
+<source>
+{
+ "status": "ok",
+ "version": "v1"
+}
+</source>
+ </section>
+ </body>
+</document>
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+ <header>
+ <title>GET version </title>
+ </header>
+
+ <body>
+ <section>
+ <title>Description</title>
+ <p>Returns a list of supported versions and the current version.</p>
+ </section>
+
+ <section>
+ <title>URL</title>
+ <p><code>http://</code>www.myserver.com<code>/templeton/v1/version</code></p>
+ </section>
+
+ <section>
+ <title>Parameters</title>
+ <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+ are accepted.</p>
+ </section>
+
+ <section>
+ <title>Results</title>
+ <table>
+ <tr><th>Name</th><th>Description</th></tr>
+
+ <tr><td><strong>supportedVersions</strong></td>
+ <td>A list of all supported versions.</td>
+ </tr>
+
+ <tr><td><strong>version</strong></td>
+ <td>The current version.</td>
+ </tr>
+
+ </table>
+ </section>
+
+ <section>
+ <title>Example</title>
+
+ <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/version'
+</source>
+
+ <p><strong>JSON Output</strong></p>
+<source>
+{
+ "supportedVersions": [
+ "v1"
+ ],
+ "version": "v1"
+}
+</source>
+ </section>
+ </body>
+</document>
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1365722&r1=1365721&r2=1365722&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Wed Jul 25 20:29:44 2012
@@ -160,7 +160,7 @@ public class MultiOutputFormat extends O
* for multiple output formats.
*
* @param job the mapreduce job to be submitted
- * @return
+ * @return JobConfigurer
*/
public static JobConfigurer createConfigurer(Job job) {
return JobConfigurer.create(job);
@@ -361,7 +361,7 @@ public class MultiOutputFormat extends O
*
* @param alias the name used for the OutputFormat during
* addOutputFormat
- * @return
+ * @return Job
*/
public Job getJob(String alias) {
Job copy = outputConfigs.get(alias);
Added: incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt Wed Jul 25 20:29:44 2012
@@ -0,0 +1,104 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+End to end tests
+---------------
+End to end tests in templeton runs tests against an existing templeton server.
+It runs hcat, mapreduce, streaming, hive and pig tests.
+
+Test cases
+----------
+The tests are defined in src/test/e2e/templeton/tests/*.conf
+
+Test framework
+--------------
+The test framework is derived from the one used in pig, there is more documentation here on the framework -
+https://cwiki.apache.org/confluence/display/PIG/HowToTest
+
+
+Setup
+-----
+1. Templeton needs to be installed and setup to be able to run hcat, maprduce, hive and pig commands.
+
+2. Install perl and following perl modules (cpan -i <MODULE_NAME>)
+* IPC::Run
+* JSON
+* Data::Dump
+* Number::Compare
+* Text::Glob
+* Data::Compare
+* File::Find::Rule
+* HTTP::Daemon
+
+Tips:
+* Using perlbrew (http://perlbrew.pl) should make installing perl modules easier.
+* Use 'yes | cpan -i <MODULE_NAME>' to avoid answering the 100's of questions cpan asks.
+
+
+
+3. Copy contents of src/test/e2e/templeton/inpdir to hdfs
+
+
+Running the tests
+-----------------
+Use the following command to run tests -
+
+ant test -Dinpdir.hdfs=<location of inpdir on hdfs> -Dtest.user.name=<user the tests should run as> \
+ -Dsecure.mode=<yes/no> -Dharness.webhdfs.url=<webhdfs url upto port num> -Dharness.templeton.url=<templeton url upto port num>
+
+If you want to run specific test group you can specify the group, for example: -Dtests.to.run='-t TestHive'
+
+If you want to run specific test in a group group you can specify the test, for example: -Dtests.to.run='-t TestHive_1'
+
+
+Running the hcat authorization tests
+------------------------------------
+Hcat authorization tests run commands as different users to test if authorization is done right.
+
+ant test-hcat-authorization -Dkeytab.dir=<keytab files dir>
+ -Dsecure.mode=<yes/no> -Dtest.group.name=<common group> -Dinpdir.hdfs=<location of inpdir on hdfs>
+ -Dtest.user.name=<user 1 belonging to common group> -Dtest.group.user.name=<user 2 belonging to common group>
+ -Dtest.other.user.name=<user 3 who does not belong to common group>
+ -Dharness.webhdfs.url=<webhdfs url upto port num> -Dharness.templeton.url=<templeton url upto port num>
+
+The <keytab files dir> is expected to have keytab filenames of the form - user_name.*keytab .
+
+
+Notes
+-----
+
+
+
+Enable webhdfs by adding the following to your hadoop hdfs-site.xml :
+
+<property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>true</value>
+</property>
+<property>
+ <name>dfs.http.address</name>
+ <value>127.0.0.1:8085</value>
+ <final>true</final>
+</property>
+
+You can build a server that will measure test coverage by using templeton:
+ant clean; ant e2e
+This assumes you've got webhdfs at the address above, the inpdir info in /user/templeton, and templeton running on the default port. You can change any of those properties in the build file.
+
+It's best to set HADOOP_HOME_WARN_SUPPRESS=true everywhere you can.
Added: incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,95 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="TestHarnessTempletonTests" default="test">
+
+ <!-- Separate property name for udfs' build.xml -->
+ <property name="e2e.lib.dir" value="${basedir}/lib"/>
+
+ <property name="test.src" value="${basedir}/tests"/>
+ <property name="driver.src" value="${basedir}/drivers"/>
+ <property name="harness.dir" value="${basedir}/../harness"/>
+ <property name="inpdir.local" value="${basedir}/inpdir/"/>
+ <property name="test.location" value="${basedir}/testWorkDir"/>
+ <property name="driver.src" value="${basedir}/drivers"/>
+
+ <!-- Check that the necessary properties are setup -->
+ <target name="property-check">
+ <fail message="Please set the property harness.webhdfs.url to the namenode base url of the cluster"
+ unless="harness.webhdfs.url"/>
+ <fail message="Please set the property harness.templeton.url to the templeton server base url of the cluster"
+ unless="harness.templeton.url"/>
+ <fail message="Please set the property inpdir.hdfs to the test input directory on hdfs"
+ unless="inpdir.hdfs"/>
+ </target>
+
+ <!-- Prep the test area -->
+ <target name="init-test">
+ <mkdir dir="${test.location}"/>
+ </target>
+
+ <target name="test" depends="property-check, init-test" >
+ <property name="tests.to.run" value=""/>
+ <exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="${harness.dir}"/>
+ <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
+ <env key="TH_WORKING_DIR" value="${test.location}"/>
+ <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+ <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+ <env key="TH_OUT" value="."/>
+ <env key="TH_ROOT" value="."/>
+ <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+ <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+ <env key="USER_NAME" value="${test.user.name}"/>
+ <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+ <env key="SECURE_MODE" value="${secure.mode}"/>
+ <arg line="${tests.to.run}"/>
+ <arg value="${basedir}/tests/serverstatus.conf"/>
+ <arg value="${basedir}/tests/ddl.conf"/>
+ <arg value="${basedir}/tests/jobsubmission.conf"/>
+ </exec>
+ </target>
+
+ <target name="test-hcat-authorization" depends="property-check, init-test" >
+ <property name="tests.to.run" value=""/>
+ <exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="${harness.dir}"/>
+ <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
+ <env key="TH_WORKING_DIR" value="${test.location}"/>
+ <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+ <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+ <env key="TH_OUT" value="."/>
+ <env key="TH_ROOT" value="."/>
+ <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+ <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+ <env key="USER_NAME" value="${test.user.name}"/>
+ <env key="GROUP_NAME" value="${test.group.name}"/>
+ <env key="GROUP_USER_NAME" value="${test.group.user.name}"/>
+ <env key="OTHER_USER_NAME" value="${test.other.user.name}"/>
+ <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+ <env key="SECURE_MODE" value="${secure.mode}"/>
+ <env key="KEYTAB_DIR" value="${keytab.dir}"/>
+ <arg line="${tests.to.run}"/>
+ <arg value="${basedir}/tests/hcatperms.conf"/>
+ </exec>
+ </target>
+
+
+
+</project>
+
+
Added: incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf Wed Jul 25 20:29:44 2012
@@ -0,0 +1,40 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+my $me = `whoami`;
+chomp $me;
+
+# The contents of this file can be rewritten to fit your installation.
+# Also, you can define the following environment variables and set things up as in the test setup
+# TH_ROOT Root directory where test harness is installed
+# TH_OUT Root directory where output data will be stored
+
+$cfg = {
+ #HDFS
+ 'outpathbase' => '/tmp/templeton_test_out'
+ , 'localoutpathbase' => "$ENV{TH_WORKING_DIR}/"
+
+ #TEST
+ , 'tmpPath' => '/tmp/pigtest'
+
+ , 'userhomePath' => "$ENV{HOME}"
+ ,'local.bin' => '/usr/bin'
+
+ ,'logDir' => "$ENV{TH_OUT}/log"
+# ,'propertiesFile' => "./conf/testpropertiesfile.conf"
+ ,'harness.console.level' => 'ERROR'
+
+};
Added: incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl Wed Jul 25 20:29:44 2012
@@ -0,0 +1,158 @@
+#!/usr/local/bin/perl -w
+use strict;
+
+my $TESTDIR = '/tmp/templetontest/';
+my $TEST_INP_DIR = '/tmp/test_inpdir/'; #dir on hadoop
+my $TEST_USER = 'hortonth';
+my $WEBHDFS_URL = 'http://localhost:50070';
+my $TEMPLETON_URL = 'http://localhost:8080';
+my $CATALINA_HOME = $ENV{'CATALINA_HOME'};
+
+#use env variables if they have been set
+if(defined $ENV{'TESTDIR'}){
+ $TESTDIR = $ENV{'TESTDIR'};
+}
+if(defined $ENV{'TEST_INP_DIR'}){
+ $TEST_INP_DIR = $ENV{'TEST_INP_DIR'};
+}
+if(defined $ENV{'TEST_USER'}){
+ $TEST_USER = $ENV{'TEST_USER'};
+}
+if(defined $ENV{'WEBHDFS_URL'}){
+ $WEBHDFS_URL = $ENV{'WEBHDFS_URL'};
+}
+if(defined $ENV{'TEMPLETON_URL'}){
+ $TEMPLETON_URL = $ENV{'TEMPLETON_URL'};
+}
+
+if(! defined $ENV{'HCAT_PREFIX'}){
+ $ENV{'HCAT_PREFIX'}='/usr/';
+}
+
+if(! defined $ENV{'HADOOP_PREFIX'}){
+ $ENV{'HADOOP_PREFIX'}='/usr/';
+}
+
+my $host = `hostname` ;
+chomp $host;
+
+if(! defined $ENV{'ZOOKEEPER_HOST'}){
+ $ENV{'ZOOKEEPER_HOST'} = $host . ':2181';
+}
+
+if(! defined $ENV{'METASTORE_HOST'}){
+ $ENV{'METASTORE_HOST'} = $host . ':9933';
+}
+
+print STDERR "##################################################################\n";
+print STDERR "Using the following settings for environment variables\n" .
+ " (Set them to override the default values) \n" .
+ "WEBHDFS_URL : $WEBHDFS_URL \n" .
+ "TEMPLETON_URL : $TEMPLETON_URL \n" .
+ 'CATALINA_HOME :' . $ENV{'CATALINA_HOME'} . "\n" .
+ 'HADOOP_PREFIX :' . $ENV{'HADOOP_PREFIX'} . "\n" .
+ 'HCAT_PREFIX :' . $ENV{'HCAT_PREFIX'} . "\n" .
+ 'ZOOKEEPER_HOST :' . $ENV{'ZOOKEEPER_HOST'} . "\n" .
+ 'METASTORE_HOST :' . $ENV{'METASTORE_HOST'} . "\n"
+;
+print STDERR "##################################################################\n";
+
+system("rm -rf $TESTDIR/");
+
+#restart tomcat with updated env variables
+my $templeton_src = "$TESTDIR/templeton_src";
+$ENV{'TEMPLETON_HOME'} = "$templeton_src/templeton";
+
+system ("$CATALINA_HOME/bin/shutdown.sh") == 0 or die "tomcat shutdown failed" ;
+sleep 3;
+
+
+
+#get templeton git repo, build and install
+system("mkdir -p $templeton_src") == 0 or die "could not create dir $templeton_src: $!";
+chdir "$templeton_src" or die "could not change directory to $templeton_src : $!";
+system ('git clone git@github.com:hortonworks/templeton.git') == 0 or die "could not clone templeton git repo";
+chdir 'templeton' or die 'could not change dir : $!';
+
+
+#put a templeton-site.xml in $TEMPLETON_HOME with zookeeper hostname
+writeTempletonSiteXml();
+system ('ant install-war') == 0 or die "templeton build failed";
+
+#tomcat should have shutdown by now, try starting it
+system ("$CATALINA_HOME/bin/startup.sh") == 0 or die 'tomcat startup failed';
+sleep 3;
+
+my $tdir = "$templeton_src/templeton/src/test/e2e/templeton";
+chdir $tdir or die "could not change dir $tdir : $!";
+
+#copy input files
+system("hadoop fs -rmr $TEST_INP_DIR");
+system("hadoop fs -copyFromLocal $tdir/inpdir $TEST_INP_DIR") == 0 or die "failed to copy input dir : $!";
+system("hadoop fs -chmod -R 777 $TEST_INP_DIR") == 0 or die "failed to set input dir permissions : $!";
+
+#start tests
+my $cmd = "ant test -Dinpdir.hdfs=$TEST_INP_DIR -Dtest.user.name=$TEST_USER" .
+ " -Dharness.webhdfs.url=$WEBHDFS_URL -Dharness.templeton.url=$TEMPLETON_URL ";
+
+system($cmd) == 0 or die "templeton tests failed";
+
+#############################
+sub writeTempletonSiteXml {
+ my $conf = $ENV{'TEMPLETON_HOME'} . "/templeton-site.xml";
+ open ( CFH, ">$conf" ) or die $!;
+
+
+ print CFH '<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <property>
+ <name>templeton.zookeeper.hosts</name>
+ <value>' .
+ $ENV{'ZOOKEEPER_HOST'} .
+'</value>
+ <description>ZooKeeper servers, as comma separated host:port pairs</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.properties</name>
+ <value>hive.metastore.local=false,hive.metastore.uris=thrift://' .
+ $ENV{'METASTORE_HOST'} .
+ ',hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true</value>
+ <description>Properties to set when running hive.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.archive</name>
+ <value>hdfs:///user/templeton/hcatalog-0.3.0.tar.gz</value>
+ <description>The path to the Hive archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.path</name>
+ <value>hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive</value>
+ <description>The path to the Hive executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.archive</name>
+ <value>hdfs:///user/templeton/pig-0.9.2.tar.gz</value>
+ <description>The path to the Pig archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.path</name>
+ <value>pig-0.9.2.tar.gz/pig-0.9.2/bin/pig</value>
+ <description>The path to the Pig executable.</description>
+ </property>
+
+</configuration>
+';
+ close CFH or die $!;
+
+;
+
+
+
+
+
+}
Propchange: incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl
------------------------------------------------------------------------------
svn:executable = *