You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2013/10/22 19:59:08 UTC
svn commit: r1534711 [3/15] - in /hive/branches/maven: ./
ant/src/org/apache/hadoop/hive/ant/
beeline/src/java/org/apache/hive/beeline/ bin/ bin/ext/
cli/src/java/org/apache/hadoop/hive/cli/ common/
common/src/java/org/apache/hadoop/hive/common/type/ c...
Modified: hive/branches/maven/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/maven/eclipse-templates/.classpath?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/eclipse-templates/.classpath (original)
+++ hive/branches/maven/eclipse-templates/.classpath Tue Oct 22 17:58:59 2013
@@ -20,7 +20,7 @@
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/ant-1.6.5.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-cli-1.2.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-codec-1.3.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-codec-1.4.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-el-1.0.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-httpclient-3.0.1.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/commons-logging-1.1.1.jar"/>
@@ -88,6 +88,7 @@
<classpathentry kind="lib" path="build/ivy/lib/default/mockito-all-@mockito-all.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/ST4-@ST4.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/snappy-@snappy.version@.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/groovy-all-@groovy.version@.jar"/>
<classpathentry kind="lib" path="build/beeline/hive-beeline-@HIVE_VERSION@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/tempus-fugit-@tempus-fugit.version@.jar"/>
<classpathentry kind="src" path="build/contrib/test/src"/>
@@ -95,6 +96,7 @@
<classpathentry kind="lib" path="build/testutils/hive-testutils-@HIVE_VERSION@.jar"/>
<classpathentry kind="src" path="build/ql/test/src"/>
<classpathentry kind="src" path="build/ql/gen/antlr/gen-java"/>
+ <classpathentry kind="src" path="build/ql/gen/vector"/>
<classpathentry kind="src" path="beeline/src/java"/>
<classpathentry kind="src" path="beeline/src/test"/>
<classpathentry kind="src" path="cli/src/java"/>
Modified: hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Tue Oct 22 17:58:59 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.index.I
import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
@@ -410,8 +411,8 @@ public class HBaseStorageHandler extends
hbaseSerde.getStorageFormatOfCol(keyColPos).get(0));
List<IndexSearchCondition> searchConditions =
new ArrayList<IndexSearchCondition>();
- ExprNodeDesc residualPredicate =
- analyzer.analyzePredicate(predicate, searchConditions);
+ ExprNodeGenericFuncDesc residualPredicate =
+ (ExprNodeGenericFuncDesc)analyzer.analyzePredicate(predicate, searchConditions);
int scSize = searchConditions.size();
if (scSize < 1 || 2 < scSize) {
// Either there was nothing which could be pushed down (size = 0),
Modified: hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hive/branches/maven/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Tue Oct 22 17:58:59 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.index.I
import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ByteStream;
@@ -253,8 +254,8 @@ public class HiveHBaseTableInputFormat e
if (filterExprSerialized == null) {
return scan;
}
- ExprNodeDesc filterExpr =
- Utilities.deserializeExpression(filterExprSerialized, jobConf);
+ ExprNodeGenericFuncDesc filterExpr =
+ Utilities.deserializeExpression(filterExprSerialized);
String colName = jobConf.get(serdeConstants.LIST_COLUMNS).split(",")[iKey];
String colType = jobConf.get(serdeConstants.LIST_COLUMN_TYPES).split(",")[iKey];
Modified: hive/branches/maven/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/maven/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm (original)
+++ hive/branches/maven/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm Tue Oct 22 17:58:59 2013
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.cli;
import junit.framework.Test;
Modified: hive/branches/maven/hcatalog/bin/hcat
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/bin/hcat?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/bin/hcat (original)
+++ hive/branches/maven/hcatalog/bin/hcat Tue Oct 22 17:58:59 2013
@@ -129,6 +129,19 @@ for jar in ${HIVE_LIB_DIR}/*.jar ; do
HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$jar
done
+# add the auxillary jars such as serdes
+if [ -d "${HIVE_AUX_JARS_PATH}" ]; then
+ for f in ${HIVE_AUX_JARS_PATH}/*.jar; do
+ if [[ ! -f $f ]]; then
+ continue;
+ fi
+ HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:$f
+ done
+elif [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
+ HIVE_AUX_JARS_PATH=`echo $HIVE_AUX_JARS_PATH | sed 's/,/:/g'`
+ HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HIVE_AUX_JARS_PATH
+fi
+
# Put external jars, hcat jar, and config file in the classpath
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${HCAT_CLASSPATH}:${HCAT_JAR}:${HIVE_CONF_DIR}
Modified: hive/branches/maven/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java (original)
+++ hive/branches/maven/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java Tue Oct 22 17:58:59 2013
@@ -271,7 +271,11 @@ public class HCatSemanticAnalyzer extend
DropDatabaseDesc dropDb = work.getDropDatabaseDesc();
if (dropDb != null) {
Database db = cntxt.getHive().getDatabase(dropDb.getDatabaseName());
- authorize(db, Privilege.DROP);
+ if (db != null){
+ // if above returned a null, then the db does not exist - probably a
+ // "drop database if exists" clause - don't try to authorize then.
+ authorize(db, Privilege.DROP);
+ }
}
DescDatabaseDesc descDb = work.getDescDatabaseDesc();
Modified: hive/branches/maven/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Tue Oct 22 17:58:59 2013
@@ -184,7 +184,8 @@ sub runTest
$testCmd->{'group'} . "_" . $testCmd->{'num'} . ".$i.out";
$tableName = $results[$i];
$modifiedTestCmd{'num'} = $testCmd->{'num'} . "_" . $i . "_benchmark";
- $modifiedTestCmd{'pig'} = "a = load '$tableName' using org.apache.hive.hcatalog.pig.HCatLoader(); store a into ':OUTPATH:';";
+ $tableLoader = (defined $testCmd->{'result_table_loader'} ? $testCmd->{'result_table_loader'} : "org.apache.hive.hcatalog.pig.HCatLoader()");
+ $modifiedTestCmd{'pig'} = "a = load '$tableName' using $tableLoader; store a into ':OUTPATH:';";
my $r = $self->runPig(\%modifiedTestCmd, $log, 1, 1);
$outputs[$i] = $r->{'output'};
} else {
Modified: hive/branches/maven/hcatalog/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/hcatalog/tests/pig.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/hcatalog/tests/pig.conf Tue Oct 22 17:58:59 2013
@@ -330,6 +330,7 @@ exec
e = load 'pig_hbase_1' using org.apache.hcatalog.pig.HCatLoader();
store e into ':OUTPATH:';\,
,'result_table' => ['pig_hbase_1','?']
+ ,'result_table_loader' => 'org.apache.hcatalog.pig.HCatLoader()'
,'sql' => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k group by name;', 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k group by name;' ]
,'floatpostprocess' => 1
,'delimiter' => ' '
@@ -350,6 +351,7 @@ d = foreach c generate name as key, (cha
store d into 'pig_hbase_2_1' using org.apache.hcatalog.pig.HCatStorer();
store d into 'pig_hbase_2_2' using org.apache.hcatalog.pig.HCatStorer();\,
,'result_table' => ['pig_hbase_2_1','pig_hbase_2_2']
+ ,'result_table_loader' => 'org.apache.hcatalog.pig.HCatLoader()'
,'sql' => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k group by name;', 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k group by name;']
,'floatpostprocess' => 1
,'delimiter' => ' '
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/README.txt?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/README.txt (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/README.txt Tue Oct 22 17:58:59 2013
@@ -87,15 +87,15 @@ Tips:
3. Copy contents of src/test/e2e/templeton/inpdir to hdfs
(e.g. ./bin/hadoop fs -put ~/dev/hive/hcatalog/src/test/e2e/templeton/inpdir/ webhcate2e)
-4. You will need to two jars in the same HDFS directory as the contents of inpdir. piggybank.jar, which can
-be obtained from Pig. The second is the hadoop-examples.jar, which can be obtained from your Hadoop distribution.
-This should be called hexamples.jar when it is uploaded to HDFS.
+4. You will need to copy three jars in the same HDFS directory as the contents of inpdir. piggybank.jar, which can
+be obtained from Pig and the other two are obtained from your Hadoop distribution.
+For Hadoop 1.x you would need to upload hadoop-examples.jar twice to HDFS one as hclient.jar and other as hexamples.jar.
+For Hadoop 2.x you would need to upload hadoop-mapreduce-client-jobclient.jar to HDFS as hclient.jar and hadoop-mapreduce-examples.jar to HDFS as hexamples.jar.
Also see http://hive.apache.org/docs/hcat_r0.5.0/rest_server_install.html#Hadoop+Distributed+Cache for notes on
additional JAR files to copy to HDFS.
5. Make sure TEMPLETON_HOME evnironment variable is set
-
6. hadoop/conf/core-site.xml should have items described in
http://hive.apache.org/docs/hcat_r0.5.0/rest_server_install.html#Permissions
@@ -104,6 +104,7 @@ Running the tests
Use the following command to run tests -
ant test -Dinpdir.hdfs=<location of inpdir on hdfs> -Dtest.user.name=<user the tests should run as> \
+ -Dtest.other.user.name=<another user who can launch job> \
-Dsecure.mode=<yes/no> -Dharness.webhdfs.url=<webhdfs url upto port num> -Dharness.templeton.url=<templeton url upto port num>
If you want to run specific test group you can specify the group, for example: -Dtests.to.run='-t TestHive'
@@ -111,6 +112,8 @@ If you want to run specific test group y
If you want to run specific test in a group group you can specify the test, for example: -Dtests.to.run='-t TestHive_1'
For example, tests/ddl.conf has several groups such as 'name' => 'REST_DDL_TABLE_BASIC'; use REST_DDL_TABLE_BASIC as the name
+If you are running with Hadoop 2, please use the flag: -Dhadoopversion=23
+
Running the hcat authorization tests
------------------------------------
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/build.xml?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/build.xml Tue Oct 22 17:58:59 2013
@@ -106,8 +106,10 @@
<env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
<env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
<env key="USER_NAME" value="${test.user.name}"/>
+ <env key="OTHER_USER_NAME" value="${test.other.user.name}"/>
<env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
<env key="SECURE_MODE" value="${secure.mode}"/>
+ <env key="HADOOP_VERSION" value="${hadoopversion}"/>
<arg line="${tests.to.run}"/>
<arg value="${basedir}/tests/serverstatus.conf"/>
<arg value="${basedir}/tests/jobsubmission_streaming.conf"/>
@@ -125,8 +127,8 @@
<property name="fork.factor.conf.file" value="1"/>
<property name="e2e.debug" value="false"/>
<property name="tests.to.run" value=""/>
- <exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
- <env key="HARNESS_ROOT" value="${harness.dir}"/>
+ <exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="."/>
<env key="DRIVER_ROOT" value="${basedir}/drivers"/>
<env key="TH_WORKING_DIR" value="${test.location}"/>
<env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/conf/default.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/conf/default.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/conf/default.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/conf/default.conf Tue Oct 22 17:58:59 2013
@@ -38,5 +38,5 @@ $cfg = {
,'logDir' => "$ENV{TH_OUT}/log"
# ,'propertiesFile' => "./conf/testpropertiesfile.conf"
,'harness.console.level' => 'ERROR'
-
+ ,'hadoopversion' => "$ENV{HADOOP_VERSION}"
};
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Tue Oct 22 17:58:59 2013
@@ -207,6 +207,17 @@ sub globalSetup
die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
+ my $testCmdBasics = $self->copyTestBasicConfig($globalHash);
+ $testCmdBasics->{'method'} = 'PUT';
+ $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' . $globalHash->{'outpath'} . '?op=MKDIRS&permission=777';
+ if (!defined $globalHash->{'is_secure_mode'} || $globalHash->{'is_secure_mode'} !~ /y.*/i) {
+ $testCmdBasics->{'url'} = $testCmdBasics->{'url'} . '&user.name=' . $globalHash->{'current_user'};
+ }
+ my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log);
+ my $json = new JSON;
+ $json->utf8->decode($curl_result->{'body'})->{'boolean'} or
+ die "Cannot create hdfs directory " . $globalHash->{'outpath'} .
+ " " . "$ERRNO\n";
}
###############################################################################
@@ -249,6 +260,12 @@ sub runTest
my ($self, $testCmd, $log) = @_;
my $subName = (caller(0))[3];
+ # Check that we should run this test. If the current hadoop version
+ # is hadoop 2 and the test is marked as "ignore23", skip the test
+ if ($self->wrongExecutionMode($testCmd, $log)) {
+ my %result;
+ return \%result;
+ }
# Handle the various methods of running used in
# the original TestDrivers
@@ -607,6 +624,13 @@ sub compare
my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
my $subName = (caller(0))[3];
+ # Check that we should run this test. If the current hadoop version
+ # is hadoop 2 and the test is marked as "ignore23", skip the test
+ if ($self->wrongExecutionMode($testCmd, $log)) {
+ # Special magic value
+ return $self->{'wrong_execution_mode'};
+ }
+
my $result = 1; # until proven wrong...
if (defined $testCmd->{'status_code'}) {
my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log);
@@ -631,7 +655,7 @@ sub compare
# in the tests, we run this case with jobName = "PigLatin:loadstore.pig"
# filter $body to leave only records with this jobName
my @filtered_body = grep {($_->{detail}{profile}{jobName} eq "PigLatin:loadstore.pig")} @$body;
- my @sorted_filtered_body = sort { $a->{id} <=> $b->{id} } @filtered_body;
+ my @sorted_filtered_body = sort { $a->{id} cmp $b->{id} } @filtered_body;
$value = $path->value(\@sorted_filtered_body);
} else {
$value = $path->value($testResult->{'body'});
@@ -952,6 +976,27 @@ sub compare
return $result;
}
+##############################################################################
+# Check whether we should be running this test or not.
+#
+sub wrongExecutionMode($$)
+{
+ my ($self, $testCmd, $log) = @_;
+
+ my $wrong = 0;
+
+ if (defined $testCmd->{'ignore23'} && $testCmd->{'hadoopversion'}=='23') {
+ $wrong = 1;
+ }
+
+ if ($wrong) {
+ print $log "Skipping test $testCmd->{'group'}" . "_" .
+ $testCmd->{'num'} . " since it is not suppsed to be run in hadoop 23\n";
+ }
+
+ return $wrong;
+}
+
###############################################################################
sub setLocationPermGroup{
my ($self, $job_info, $testCmd, $log) = @_;
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/doas.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/doas.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/doas.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/doas.conf Tue Oct 22 17:58:59 2013
@@ -104,12 +104,12 @@ $cfg =
{
#descbe the table (as the table owner but using doAs)
- #this should succeed (it seems reading metadata is allowed even if reading data is not)
+ #this should fail when using StorageBasedAuthorizationProvider
'num' => 6,
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2/partition?user.name=:UNAME:?doAs=:DOAS:',
- 'status_code' => 200,
- 'json_field_substr_match' => {'database' => 'default', 'table' => ':UNAME:_doastab2'},
+ 'status_code' => 500,
+ 'json_field_substr_match' => {'error' => 'FAILED: AuthorizationException java\.security\.AccessControlException: action READ not permitted on path .* for user :UNAME:\?doAs=:DOAS:'},
},
{
@@ -118,7 +118,7 @@ $cfg =
'method' => 'DELETE',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/:UNAME:_doastab2?user.name=:UNAME:&doAs=:DOAS:',
'status_code' => 500,
- 'json_field_substr_match' => {'error' => 'FAILED: Execution Error, return code 1 from org\.apache\.hadoop\.hive\.ql\.exec\.DDLTask\. MetaException\(message:java\.security\.AccessControlException: action WRITE not permitted on path.* for user :DOAS:\)'},
+ 'json_field_substr_match' => {'error' => 'Authorization failed:java\.security\.AccessControlException: action WRITE not permitted on path .* for user :DOAS:'},
},
{
#descbe the table....
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/hcatperms.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/hcatperms.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/hcatperms.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/hcatperms.conf Tue Oct 22 17:58:59 2013
@@ -170,7 +170,7 @@ $cfg =
{
- #check default db permissions
+ #check db permissions
#the default group is currently the group of parent dir
'num' => 5,
@@ -188,7 +188,7 @@ $cfg =
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/hcatperms_e',
'user_name' => ':UNAME:',
'format_header' => 'Content-Type: application/json',
- 'post_options' => ['{}']
+ 'post_options' => ['{ "permissions" : "rwx------" }']
}
],
@@ -677,7 +677,7 @@ $cfg =
'format_header' => 'Content-Type: application/json',
'user_name' => ':UNAME_OTHER:',
'status_code' => 500,
- 'json_field_substr_match' => {'error' => 'unable to show tables'},
+ 'json_field_substr_match' => {'error' => 'FAILED: AuthorizationException .*\.security\.AccessControlException: action READ not permitted on path .* for user :UNAME_OTHER:'},
},
@@ -868,7 +868,7 @@ $cfg =
},
{
- #check default permissions
+ #check permissions
#the default group is currently the group of parent dir
'num' => 2,
'setup' => [
@@ -883,7 +883,7 @@ $cfg =
'method' => 'PUT',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/default/table/permstable_:TNUM:',
'format_header' => 'Content-Type: application/json',
- 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"} }'],
+ 'post_options' => ['{ "columns": [ { "name" : "i", "type" : "int" } ],"format" : { "storedAs" : "rcfile"}, "permissions" : "rwx------" }'],
'user_name' => ':UNAME:',
'status_code' => 200,
}
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Tue Oct 22 17:58:59 2013
@@ -77,24 +77,8 @@ $cfg =
'check_call_back' => 1,
},
{
-
- 'num' => 2,
- 'method' => 'POST',
- 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
- 'post_options' => ['user.name=:UNAME:','arg=-mt', 'arg=660000',
- 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=sleep', ],
- 'json_field_substr_match' => { 'id' => '\d+'},
- #results
- 'status_code' => 200,
- 'check_job_created' => 1,
- 'check_job_complete' => 'SUCCESS',
- 'check_job_exit_value' => 0,
- 'check_call_back' => 1,
- 'timeout' => 840, #increase timeout as this test takes long
- },
- {
# with log enabled
- 'num' => 3,
+ 'num' => 2,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt',
@@ -250,7 +234,7 @@ $cfg =
{
#a simple load store script with log enabled
- 'num' => 9,
+ 'num' => 10,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/pig',
'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig',
@@ -265,7 +249,31 @@ $cfg =
'check_call_back' => 1,
},
- #test 10
+ {
+ #note: this test will fail unless Hive is installed in the default location Pig expects it in
+ #HIVE-5547 will address this limitation
+ 'num' => 11,
+ 'setup' => [
+ {
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl',
+ 'status_code' => 200,
+ 'post_options' => ['user.name=:UNAME:','exec=drop table if exists hcattest_pig; create table hcattest_pig(i int, j int) STORED AS textfile;'],
+ 'json_field_substr_match' => {'stderr' => 'OK'}
+ }
+ ],
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:', 'arg=-useHCatalog', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:', 'arg=-p', 'arg= OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/hcatloadstore.pig'],
+
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ #test 11
#TODO jython test
@@ -310,6 +318,7 @@ $cfg =
{
#test show tables
'num' => 3,
+ 'ignore23' => 'Log collector does not work with Hadoop 2',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
'post_options' => ['user.name=:UNAME:','execute=drop table if exists mynums;', ],
@@ -435,7 +444,8 @@ $cfg =
{
#test add jar
- 'num' => 9,
+ 'num' => 11,
+ 'ignore23' => 'Log collector does not work with Hadoop 2',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',],
@@ -449,7 +459,7 @@ $cfg =
},
{
#test add jar when the jar is not shipped
- 'num' => 10,
+ 'num' => 12,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',],
@@ -463,7 +473,8 @@ $cfg =
},
{
#enable logs
- 'num' => 11,
+ 'num' => 13,
+ 'ignore23' => 'Log collector does not work with Hadoop 2',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
'post_options' => ['user.name=:UNAME:','execute=select a,b from mynums', 'statusdir=:OUTDIR:/status', 'enablelog=true'],
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission2.conf Tue Oct 22 17:58:59 2013
@@ -42,11 +42,12 @@ $cfg =
# see HIVE-4808
# this is a long running test, takes 11 minutes
# -mt must be greater than mapred.task.timeout (600000ms)
- 'num' => 2,
+ 'num' => 1,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
'post_options' => ['user.name=:UNAME:','arg=-mt', 'arg=642000',
- 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=sleep', 'statusdir=/tmp' ],
+ 'jar=:INPDIR_HDFS:/hclient.jar', 'class=sleep',
+ 'statusdir=/tmp/TestHeartbeat_1' ],
'json_field_substr_match' => { 'id' => '\d+'},
'status_code' => 200,
'check_job_created' => 1,
@@ -55,7 +56,7 @@ $cfg =
# it contains the value.
# 'check_job_exit_value' => 0,
'check_call_back' => 1,
- 'timeout_seconds' => 800,
+ 'timeout' => 800,
},
]
},
Modified: hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf (original)
+++ hive/branches/maven/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf Tue Oct 22 17:58:59 2013
@@ -70,6 +70,7 @@ $cfg =
{
#enable log
'num' => 3,
+ 'ignore23' => 'Log collector does not work with Hadoop 2',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
'post_options' => ['user.name=:UNAME:',
@@ -88,6 +89,7 @@ $cfg =
{
#enable log, failed job case
'num' => 4,
+ 'ignore23' => 'Log collector does not work with Hadoop 2',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt',
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh Tue Oct 22 17:58:59 2013
@@ -121,7 +121,7 @@ function check_pid() {
# Start the webhcat server in the foreground
function foreground_webhcat() {
- $start_cmd
+ exec $start_cmd
}
# Start the webhcat server in the background. Record the PID for
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/CompleteDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/CompleteDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/CompleteDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/CompleteDelegator.java Tue Oct 22 17:58:59 2013
@@ -20,11 +20,15 @@ package org.apache.hive.hcatalog.templet
import java.io.IOException;
import java.net.URL;
-import java.net.MalformedURLException;
import java.util.Date;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hive.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.templeton.tool.DelegationTokenCache;
import org.apache.hive.hcatalog.templeton.tool.JobState;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -39,11 +43,12 @@ import org.apache.hive.hcatalog.templeto
* this at the same time. That should never happen.
*
* We use a Hadoop config var to notify this class on the completion
- * of a job. Hadoop will call use multiple times in the event of
+ * of a job. Hadoop will call us multiple times in the event of
* failure. Even if the failure is that the client callback failed.
*
* See LauncherDelegator for the HADOOP_END_RETRY* vars that are set.
*/
+@InterfaceAudience.Private
public class CompleteDelegator extends TempletonDelegator {
private static final Log LOG = LogFactory.getLog(CompleteDelegator.class);
@@ -51,28 +56,36 @@ public class CompleteDelegator extends T
super(appConf);
}
- public CompleteBean run(String id)
+ public CompleteBean run(String id, String jobStatus)
throws CallbackFailedException, IOException {
if (id == null)
acceptWithError("No jobid given");
JobState state = null;
+ /* we don't want to cancel the delegation token if we think the callback is going to
+ to be retried, for example, because the job is not complete yet */
+ boolean cancelMetastoreToken = false;
try {
state = new JobState(id, Main.getAppConfigInstance());
if (state.getCompleteStatus() == null)
- failed("Job not yet complete. jobId=" + id, null);
+ failed("Job not yet complete. jobId=" + id + " Status from JT=" + jobStatus, null);
Long notified = state.getNotifiedTime();
- if (notified != null)
+ if (notified != null) {
+ cancelMetastoreToken = true;
return acceptWithError("Callback already run for jobId=" + id +
" at " + new Date(notified));
+ }
String callback = state.getCallback();
- if (callback == null)
+ if (callback == null) {
+ cancelMetastoreToken = true;
return new CompleteBean("No callback registered");
-
+ }
+
try {
doCallback(state.getId(), callback);
+ cancelMetastoreToken = true;
} catch (Exception e) {
failed("Callback failed " + callback + " for " + id, e);
}
@@ -80,8 +93,26 @@ public class CompleteDelegator extends T
state.setNotifiedTime(System.currentTimeMillis());
return new CompleteBean("Callback sent");
} finally {
- if (state != null)
- state.close();
+ state.close();
+ HiveMetaStoreClient client = null;
+ try {
+ if(cancelMetastoreToken) {
+ String metastoreTokenStrForm =
+ DelegationTokenCache.getStringFormTokenCache().getDelegationToken(id);
+ if(metastoreTokenStrForm != null) {
+ client = HCatUtil.getHiveClient(new HiveConf());
+ client.cancelDelegationToken(metastoreTokenStrForm);
+ LOG.debug("Cancelled token for jobId=" + id + " status from JT=" + jobStatus);
+ DelegationTokenCache.getStringFormTokenCache().removeDelegationToken(id);
+ }
+ }
+ }
+ catch(Exception ex) {
+ LOG.warn("Failed to cancel metastore delegation token for jobId=" + id, ex);
+ }
+ finally {
+ HCatUtil.closeHiveClientQuietly(client);
+ }
}
}
@@ -90,8 +121,7 @@ public class CompleteDelegator extends T
* finished. If the url has the string $jobId in it, it will be
* replaced with the completed jobid.
*/
- public static void doCallback(String jobid, String url)
- throws MalformedURLException, IOException {
+ public static void doCallback(String jobid, String url) throws IOException {
if (url.contains("$jobId"))
url = url.replace("$jobId", jobid);
TempletonUtils.fetchUrl(new URL(url));
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Oct 22 17:58:59 2013
@@ -65,6 +65,8 @@ public class HiveDelegator extends Launc
args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog));
args.add("--");
TempletonUtils.addCmdForWindows(args);
+ addHiveMetaStoreTokenArg();
+
args.add(appConf.hivePath());
args.add("--service");
@@ -111,9 +113,10 @@ public class HiveDelegator extends Launc
ArrayList<String> args = new ArrayList<String>();
ArrayList<String> allFiles = new ArrayList<String>();
- if (TempletonUtils.isset(srcFile))
+ if (TempletonUtils.isset(srcFile)) {
allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf,
runAs));
+ }
if (TempletonUtils.isset(otherFiles)) {
String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Oct 22 17:58:59 2013
@@ -42,14 +42,15 @@ public class JarDelegator extends Launch
public EnqueueBean run(String user, Map<String, Object> userArgs, String jar, String mainClass,
String libjars, String files,
List<String> jarArgs, List<String> defines,
- String statusdir, String callback, String completedUrl,
+ String statusdir, String callback,
+ boolean usehcatalog, String completedUrl,
boolean enablelog, JobType jobType)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(jar, mainClass,
libjars, files, jarArgs, defines,
- statusdir, completedUrl, enablelog, jobType);
+ statusdir, usehcatalog, completedUrl, enablelog, jobType);
return enqueueController(user, userArgs, callback, args);
}
@@ -57,23 +58,30 @@ public class JarDelegator extends Launch
private List<String> makeArgs(String jar, String mainClass,
String libjars, String files,
List<String> jarArgs, List<String> defines,
- String statusdir, String completedUrl,
+ String statusdir, boolean usehcatalog, String completedUrl,
boolean enablelog, JobType jobType)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
try {
- ArrayList<String> allFiles = new ArrayList();
+ ArrayList<String> allFiles = new ArrayList<String>();
allFiles.add(TempletonUtils.hadoopFsFilename(jar, appConf, runAs));
args.addAll(makeLauncherArgs(appConf, statusdir,
completedUrl, allFiles, enablelog, jobType));
args.add("--");
TempletonUtils.addCmdForWindows(args);
+
+ //check if the rest command specified explicitly to use hcatalog
+ if(usehcatalog){
+ addHiveMetaStoreTokenArg();
+ }
+
args.add(appConf.clusterHadoop());
args.add("jar");
args.add(TempletonUtils.hadoopFsPath(jar, appConf, runAs).getName());
- if (TempletonUtils.isset(mainClass))
+ if (TempletonUtils.isset(mainClass)) {
args.add(mainClass);
+ }
if (TempletonUtils.isset(libjars)) {
String libjarsListAsString =
TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs);
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Tue Oct 22 17:58:59 2013
@@ -24,10 +24,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.commons.exec.ExecuteException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -44,7 +44,8 @@ import org.apache.hive.hcatalog.templeto
public class LauncherDelegator extends TempletonDelegator {
private static final Log LOG = LogFactory.getLog(LauncherDelegator.class);
protected String runAs = null;
- static public enum JobType {JAR, STREAMING, PIG, HIVE};
+ static public enum JobType {JAR, STREAMING, PIG, HIVE}
+ private boolean secureMeatastoreAccess = false;
public LauncherDelegator(AppConfig appConf) {
super(appConf);
@@ -70,7 +71,7 @@ public class LauncherDelegator extends T
*/
public EnqueueBean enqueueController(String user, Map<String, Object> userArgs, String callback,
List<String> args)
- throws NotAuthorizedException, BusyException, ExecuteException,
+ throws NotAuthorizedException, BusyException,
IOException, QueueException {
try {
UserGroupInformation ugi = UgiFactory.getUgi(user);
@@ -82,9 +83,10 @@ public class LauncherDelegator extends T
long elapsed = ((System.nanoTime() - startTime) / ((int) 1e6));
LOG.debug("queued job " + id + " in " + elapsed + " ms");
- if (id == null)
+ if (id == null) {
throw new QueueException("Unable to get job id");
-
+ }
+
registerJob(id, user, callback, userArgs);
return new EnqueueBean(id);
@@ -95,16 +97,14 @@ public class LauncherDelegator extends T
private String queueAsUser(UserGroupInformation ugi, final List<String> args)
throws IOException, InterruptedException {
- String id = ugi.doAs(new PrivilegedExceptionAction<String>() {
+ return ugi.doAs(new PrivilegedExceptionAction<String>() {
public String run() throws Exception {
String[] array = new String[args.size()];
- TempletonControllerJob ctrl = new TempletonControllerJob();
+ TempletonControllerJob ctrl = new TempletonControllerJob(secureMeatastoreAccess);
ToolRunner.run(ctrl, args.toArray(array));
return ctrl.getSubmittedId();
}
});
-
- return id;
}
public List<String> makeLauncherArgs(AppConfig appConf, String statusdir,
@@ -182,8 +182,9 @@ public class LauncherDelegator extends T
*/
public static String makeOverrideClasspath(AppConfig appConf) {
String[] overrides = appConf.overrideJars();
- if (overrides == null)
+ if (overrides == null) {
return null;
+ }
ArrayList<String> cp = new ArrayList<String>();
for (String fname : overrides) {
@@ -204,5 +205,18 @@ public class LauncherDelegator extends T
args.add(name + "=" + val);
}
}
-
+ /**
+ * This is called by subclasses when they determined that the sumbmitted job requires
+ * metastore access (e.g. Pig job that uses HCatalog). This then determines if
+ * secure access is required and causes TempletonControllerJob to set up a delegation token.
+ * @see TempletonControllerJob
+ */
+ void addHiveMetaStoreTokenArg() {
+ //in order for this to work hive-site.xml must be on the classpath
+ HiveConf hiveConf = new HiveConf();
+ if(!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL)) {
+ return;
+ }
+ secureMeatastoreAccess = true;
+ }
}
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Oct 22 17:58:59 2013
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;
@@ -36,6 +38,7 @@ import org.apache.hive.hcatalog.templeto
* This is the backend of the pig web service.
*/
public class PigDelegator extends LauncherDelegator {
+ private static final Log LOG = LogFactory.getLog(PigDelegator.class);
public PigDelegator(AppConfig appConf) {
super(appConf);
}
@@ -43,27 +46,43 @@ public class PigDelegator extends Launch
public EnqueueBean run(String user, Map<String, Object> userArgs,
String execute, String srcFile,
List<String> pigArgs, String otherFiles,
- String statusdir, String callback, String completedUrl, boolean enablelog)
+ String statusdir, String callback,
+ boolean usehcatalog, String completedUrl, boolean enablelog)
throws NotAuthorizedException, BadParam, BusyException, QueueException,
ExecuteException, IOException, InterruptedException {
runAs = user;
List<String> args = makeArgs(execute,
srcFile, pigArgs,
- otherFiles, statusdir, completedUrl, enablelog);
+ otherFiles, statusdir, usehcatalog, completedUrl, enablelog);
return enqueueController(user, userArgs, callback, args);
}
+ /**
+ * @param execute pig query string to be executed
+ * @param srcFile pig query file to be executed
+ * @param pigArgs pig command line arguments
+ * @param otherFiles files to be copied to the map reduce cluster
+ * @param statusdir status dir location
+ * @param usehcatalog whether the command uses hcatalog/needs to connect
+ * to hive metastore server
+ * @param completedUrl call back url
+ * @return
+ * @throws BadParam
+ * @throws IOException
+ * @throws InterruptedException
+ */
private List<String> makeArgs(String execute, String srcFile,
List<String> pigArgs, String otherFiles,
- String statusdir, String completedUrl, boolean enablelog)
+ String statusdir, boolean usehcatalog,
+ String completedUrl, boolean enablelog)
throws BadParam, IOException, InterruptedException {
ArrayList<String> args = new ArrayList<String>();
try {
ArrayList<String> allFiles = new ArrayList<String>();
- if (TempletonUtils.isset(srcFile))
- allFiles.add(TempletonUtils.hadoopFsFilename
- (srcFile, appConf, runAs));
+ if (TempletonUtils.isset(srcFile)) {
+ allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf, runAs));
+ }
if (TempletonUtils.isset(otherFiles)) {
String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
allFiles.addAll(Arrays.asList(ofs));
@@ -85,6 +104,12 @@ public class PigDelegator extends Launch
for (String pigArg : pigArgs) {
args.add(TempletonUtils.quoteForWindows(pigArg));
}
+ //check if the REST command specified explicitly to use hcatalog
+ // or if it says that implicitly using the pig -useHCatalog arg
+ if(usehcatalog || hasPigArgUseHcat(pigArgs)){
+ addHiveMetaStoreTokenArg();
+ }
+
if (TempletonUtils.isset(execute)) {
args.add("-execute");
args.add(TempletonUtils.quoteForWindows(execute));
@@ -101,4 +126,12 @@ public class PigDelegator extends Launch
return args;
}
+
+ /**
+ * Check if the pig arguments has -useHCatalog set
+ * see http://hive.apache.org/docs/hcat_r0.5.0/loadstore.pdf
+ */
+ private boolean hasPigArgUseHcat(List<String> pigArgs) {
+ return pigArgs.contains("-useHCatalog");
+ }
}
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java Tue Oct 22 17:58:59 2013
@@ -33,10 +33,10 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.io.Text;
-import org.apache.thrift.TException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.thrift.TException;
/**
* Helper class to run jobs using Kerberos security. Always safe to
@@ -44,8 +44,8 @@ import org.apache.hadoop.security.token.
*/
public class SecureProxySupport {
private Path tokenPath;
- private final String HCAT_SERVICE = "hcat";
- private boolean isEnabled;
+ public static final String HCAT_SERVICE = "hcat";
+ private final boolean isEnabled;
private String user;
public SecureProxySupport() {
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Oct 22 17:58:59 2013
@@ -186,8 +186,9 @@ public class Server {
verifyDdlParam(db, ":db");
HcatDelegator d = new HcatDelegator(appConf, execService);
- if (!TempletonUtils.isset(tablePattern))
+ if (!TempletonUtils.isset(tablePattern)) {
tablePattern = "*";
+ }
return d.listTables(getDoAsUser(), db, tablePattern);
}
@@ -252,10 +253,12 @@ public class Server {
verifyDdlParam(table, ":table");
HcatDelegator d = new HcatDelegator(appConf, execService);
- if ("extended".equals(format))
+ if ("extended".equals(format)) {
return d.descExtendedTable(getDoAsUser(), db, table);
- else
+ }
+ else {
return d.descTable(getDoAsUser(), db, table, false);
+ }
}
/**
@@ -455,8 +458,9 @@ public class Server {
verifyUser();
HcatDelegator d = new HcatDelegator(appConf, execService);
- if (!TempletonUtils.isset(dbPattern))
+ if (!TempletonUtils.isset(dbPattern)) {
dbPattern = "*";
+ }
return d.listDatabases(getDoAsUser(), dbPattern);
}
@@ -508,8 +512,9 @@ public class Server {
BadParam, ExecuteException, IOException {
verifyUser();
verifyDdlParam(db, ":db");
- if (TempletonUtils.isset(option))
+ if (TempletonUtils.isset(option)) {
verifyDdlParam(option, "option");
+ }
HcatDelegator d = new HcatDelegator(appConf, execService);
return d.dropDatabase(getDoAsUser(), db, ifExists, option,
group, permissions);
@@ -579,6 +584,7 @@ public class Server {
/**
* Run a MapReduce Streaming job.
+ * @param callback URL which WebHCat will call when the hive job finishes
*/
@POST
@Path("mapreduce/streaming")
@@ -628,6 +634,11 @@ public class Server {
/**
* Run a MapReduce Jar job.
+ * Params correspond to the REST api params
+ * @param usehcatalog if {@code true}, means the Jar uses HCat and thus needs to access
+ * metastore, which requires additional steps for WebHCat to perform in a secure cluster.
+ * @param callback URL which WebHCat will call when the hive job finishes
+ * @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
*/
@POST
@Path("mapreduce/jar")
@@ -640,6 +651,7 @@ public class Server {
@FormParam("define") List<String> defines,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
+ @FormParam("usehcatalog") boolean usehcatalog,
@FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
@@ -665,11 +677,18 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
jar, mainClass,
libjars, files, args, defines,
- statusdir, callback, getCompletedUrl(), enablelog, JobType.JAR);
+ statusdir, callback, usehcatalog, getCompletedUrl(), enablelog, JobType.JAR);
}
/**
* Run a Pig job.
+ * Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usehcatalog},
+ * is interpreted as true.
+ * @param usehcatalog if {@code true}, means the Pig script uses HCat and thus needs to access
+ * metastore, which requires additional steps for WebHCat to perform in a secure cluster.
+ * This does nothing to ensure that Pig is installed on target node in the cluster.
+ * @param callback URL which WebHCat will call when the hive job finishes
+ * @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob
*/
@POST
@Path("pig")
@@ -680,12 +699,14 @@ public class Server {
@FormParam("files") String otherFiles,
@FormParam("statusdir") String statusdir,
@FormParam("callback") String callback,
+ @FormParam("usehcatalog") boolean usehcatalog,
@FormParam("enablelog") boolean enablelog)
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
- if (execute == null && srcFile == null)
+ if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
+ }
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
@@ -704,7 +725,7 @@ public class Server {
return d.run(getDoAsUser(), userArgs,
execute, srcFile,
pigArgs, otherFiles,
- statusdir, callback, getCompletedUrl(), enablelog);
+ statusdir, callback, usehcatalog, getCompletedUrl(), enablelog);
}
/**
@@ -719,7 +740,7 @@ public class Server {
* used in "add jar" statement in hive script
* @param defines shortcut for command line arguments "--define"
* @param statusdir where the stderr/stdout of templeton controller job goes
- * @param callback callback url when the hive job finishes
+ * @param callback URL which WebHCat will call when the hive job finishes
* @param enablelog whether to collect mapreduce log into statusdir/logs
*/
@POST
@@ -736,8 +757,9 @@ public class Server {
throws NotAuthorizedException, BusyException, BadParam, QueueException,
ExecuteException, IOException, InterruptedException {
verifyUser();
- if (execute == null && srcFile == null)
+ if (execute == null && srcFile == null) {
throw new BadParam("Either execute or file parameter required");
+ }
//add all function arguments to a map
Map<String, Object> userArgs = new HashMap<String, Object>();
@@ -874,10 +896,12 @@ public class Server {
@GET
@Path("internal/complete/{jobid}")
@Produces({MediaType.APPLICATION_JSON})
- public CompleteBean completeJob(@PathParam("jobid") String jobid)
+ public CompleteBean completeJob(@PathParam("jobid") String jobid,
+ @QueryParam("status") String jobStatus)
throws CallbackFailedException, IOException {
+ LOG.debug("Received callback " + theUriInfo.getRequestUri());
CompleteDelegator d = new CompleteDelegator(appConf);
- return d.run(jobid);
+ return d.run(jobid, jobStatus);
}
/**
@@ -887,8 +911,9 @@ public class Server {
String requestingUser = getRequestingUser();
if (requestingUser == null) {
String msg = "No user found.";
- if (!UserGroupInformation.isSecurityEnabled())
+ if (!UserGroupInformation.isSecurityEnabled()) {
msg += " Missing " + PseudoAuthenticator.USER_NAME + " parameter.";
+ }
throw new NotAuthorizedException(msg);
}
if(doAs != null && !doAs.equals(requestingUser)) {
@@ -897,9 +922,10 @@ public class Server {
ProxyUserSupport.validate(requestingUser, getRequestingHost(requestingUser, request), doAs);
}
}
+
/**
* All 'tasks' spawned by WebHCat should be run as this user. W/o doAs query parameter
- * this is just the user making the request (or
+ * this is just the user making the request (or
* {@link org.apache.hadoop.security.authentication.client.PseudoAuthenticator#USER_NAME}
* query param).
* @return value of doAs query parameter or {@link #getRequestingUser()}
@@ -912,8 +938,9 @@ public class Server {
*/
public void verifyParam(String param, String name)
throws BadParam {
- if (param == null)
+ if (param == null) {
throw new BadParam("Missing " + name + " parameter");
+ }
}
/**
@@ -921,8 +948,9 @@ public class Server {
*/
public void verifyParam(List<String> param, String name)
throws BadParam {
- if (param == null || param.isEmpty())
+ if (param == null || param.isEmpty()) {
throw new BadParam("Missing " + name + " parameter");
+ }
}
public static final Pattern DDL_ID = Pattern.compile("[a-zA-Z]\\w*");
@@ -937,8 +965,9 @@ public class Server {
throws BadParam {
verifyParam(param, name);
Matcher m = DDL_ID.matcher(param);
- if (!m.matches())
+ if (!m.matches()) {
throw new BadParam("Invalid DDL identifier " + name);
+ }
}
/**
* Get the user name from the security context, i.e. the user making the HTTP request.
@@ -946,10 +975,12 @@ public class Server {
* value of user.name query param, in kerberos mode it's the kinit'ed user.
*/
private String getRequestingUser() {
- if (theSecurityContext == null)
+ if (theSecurityContext == null) {
return null;
- if (theSecurityContext.getUserPrincipal() == null)
+ }
+ if (theSecurityContext.getUserPrincipal() == null) {
return null;
+ }
//map hue/foo.bar@something.com->hue since user group checks
// and config files are in terms of short name
return UserGroupInformation.createRemoteUser(
@@ -960,16 +991,18 @@ public class Server {
* The callback url on this server when a task is completed.
*/
public String getCompletedUrl() {
- if (theUriInfo == null)
+ if (theUriInfo == null) {
return null;
- if (theUriInfo.getBaseUri() == null)
+ }
+ if (theUriInfo.getBaseUri() == null) {
return null;
+ }
return theUriInfo.getBaseUri() + VERSION
- + "/internal/complete/$jobId";
+ + "/internal/complete/$jobId?status=$jobStatus";
}
/**
- * Returns canonical host name from which the request is made; used for doAs validation
+ * Returns canonical host name from which the request is made; used for doAs validation
*/
private static String getRequestingHost(String requestingUser, HttpServletRequest request) {
final String unkHost = "???";
@@ -998,7 +1031,7 @@ public class Server {
}
private void checkEnableLogPrerequisite(boolean enablelog, String statusdir) throws BadParam {
- if (enablelog == true && !TempletonUtils.isset(statusdir))
+ if (enablelog && !TempletonUtils.isset(statusdir))
throw new BadParam("enablelog is only applicable when statusdir is set");
}
}
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java Tue Oct 22 17:58:59 2013
@@ -58,7 +58,7 @@ public class StreamingDelegator extends
return d.run(user, userArgs,
appConf.streamingJar(), null,
null, files, args, defines,
- statusdir, callback, completedUrl, enableLog, jobType);
+ statusdir, callback, false, completedUrl, enableLog, jobType);
}
private List<String> makeArgs(List<String> inputs,
Modified: hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/branches/maven/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Oct 22 17:58:59 2013
@@ -25,6 +25,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -41,6 +42,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
@@ -55,9 +58,11 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.templeton.BadParam;
import org.apache.hive.hcatalog.templeton.LauncherDelegator;
+import org.apache.hive.hcatalog.templeton.SecureProxySupport;
+import org.apache.hive.hcatalog.templeton.UgiFactory;
+import org.apache.thrift.TException;
/**
* A Map Reduce job that will start another job.
@@ -70,6 +75,13 @@ import org.apache.hive.hcatalog.templeto
* - run a keep alive thread so the job doesn't end.
* - Optionally, store the stdout, stderr, and exit value of the child
* in hdfs files.
+ *
+ * A note on security. When jobs are submitted through WebHCat that use HCatalog, it means that
+ * metastore access is required. Hive queries, of course, need metastore access. This in turn
+ * requires delegation token to be obtained for metastore in a <em>secure cluster</em>. Since we
+ * can't usually parse the job to find out if it is using metastore, we require 'usehcatalog'
+ * parameter supplied in the REST call. WebHcat takes care of cancelling the token when the job
+ * is complete.
*/
public class TempletonControllerJob extends Configured implements Tool {
public static final String COPY_NAME = "templeton.copy";
@@ -89,12 +101,19 @@ public class TempletonControllerJob exte
public static final String TOKEN_FILE_ARG_PLACEHOLDER
= "__WEBHCAT_TOKEN_FILE_LOCATION__";
-
private static TrivialExecService execService = TrivialExecService.getInstance();
private static final Log LOG = LogFactory.getLog(TempletonControllerJob.class);
+ private final boolean secureMetastoreAccess;
-
+ /**
+ * @param secureMetastoreAccess - if true, a delegation token will be created
+ * and added to the job
+ */
+ public TempletonControllerJob(boolean secureMetastoreAccess) {
+ super();
+ this.secureMetastoreAccess = secureMetastoreAccess;
+ }
public static class LaunchMapper
extends Mapper<NullWritable, NullWritable, Text, Text> {
protected Process startJob(Context context, String user,
@@ -194,8 +213,9 @@ public class TempletonControllerJob exte
proc.waitFor();
keepAlive.sendReport = false;
pool.shutdown();
- if (!pool.awaitTermination(WATCHER_TIMEOUT_SECS, TimeUnit.SECONDS))
+ if (!pool.awaitTermination(WATCHER_TIMEOUT_SECS, TimeUnit.SECONDS)) {
pool.shutdownNow();
+ }
writeExitValue(conf, proc.exitValue(), statusdir);
JobState state = new JobState(context.getJobID().toString(), conf);
@@ -210,11 +230,13 @@ public class TempletonControllerJob exte
logRetriever.run();
}
- if (proc.exitValue() != 0)
+ if (proc.exitValue() != 0) {
System.err.println("templeton: job failed with exit code "
+ proc.exitValue());
- else
+ }
+ else {
System.err.println("templeton: job completed with exit code 0");
+ }
}
private void executeWatcher(ExecutorService pool, Configuration conf,
@@ -248,10 +270,10 @@ public class TempletonControllerJob exte
}
private static class Watcher implements Runnable {
- private InputStream in;
+ private final InputStream in;
private OutputStream out;
- private JobID jobid;
- private Configuration conf;
+ private final JobID jobid;
+ private final Configuration conf;
public Watcher(Configuration conf, JobID jobid, InputStream in,
String statusdir, String name)
@@ -341,21 +363,26 @@ public class TempletonControllerJob exte
private JobID submittedJobId;
public String getSubmittedId() {
- if (submittedJobId == null)
+ if (submittedJobId == null) {
return null;
- else
+ }
+ else {
return submittedJobId.toString();
+ }
}
/**
* Enqueue the job and print out the job id for later collection.
+ * @see org.apache.hive.hcatalog.templeton.CompleteDelegator
*/
@Override
public int run(String[] args)
- throws IOException, InterruptedException, ClassNotFoundException {
+ throws IOException, InterruptedException, ClassNotFoundException, TException {
Configuration conf = getConf();
+
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args));
- conf.set("user.name", UserGroupInformation.getCurrentUser().getShortUserName());
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ conf.set("user.name", user);
Job job = new Job(conf);
job.setJarByClass(TempletonControllerJob.class);
job.setJobName("TempletonControllerJob");
@@ -363,8 +390,7 @@ public class TempletonControllerJob exte
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setInputFormatClass(SingleInputFormat.class);
- NullOutputFormat<NullWritable, NullWritable> of
- = new NullOutputFormat<NullWritable, NullWritable>();
+ NullOutputFormat<NullWritable, NullWritable> of = new NullOutputFormat<NullWritable, NullWritable>();
job.setOutputFormatClass(of.getClass());
job.setNumReduceTasks(0);
@@ -372,18 +398,51 @@ public class TempletonControllerJob exte
Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token"));
job.getCredentials().addToken(new Text("mr token"), mrdt);
+
+ String metastoreTokenStrForm = addHMSToken(job, user);
+
job.submit();
submittedJobId = job.getJobID();
+ if(metastoreTokenStrForm != null) {
+ //so that it can be cancelled later from CompleteDelegator
+ DelegationTokenCache.getStringFormTokenCache().storeDelegationToken(
+ submittedJobId.toString(), metastoreTokenStrForm);
+ LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + " " +
+ "user=" + user);
+ }
return 0;
}
-
-
- public static void main(String[] args) throws Exception {
- int ret = ToolRunner.run(new TempletonControllerJob(), args);
- if (ret != 0)
- System.err.println("TempletonControllerJob failed!");
- System.exit(ret);
+ private String addHMSToken(Job job, String user) throws IOException, InterruptedException,
+ TException {
+ if(!secureMetastoreAccess) {
+ return null;
+ }
+ Token<org.apache.hadoop.hive.thrift.DelegationTokenIdentifier> hiveToken =
+ new Token<org.apache.hadoop.hive.thrift.DelegationTokenIdentifier>();
+ String metastoreTokenStrForm = buildHcatDelegationToken(user);
+ hiveToken.decodeFromUrlString(metastoreTokenStrForm);
+ job.getCredentials().addToken(new
+ Text(SecureProxySupport.HCAT_SERVICE), hiveToken);
+ return metastoreTokenStrForm;
+ }
+ private String buildHcatDelegationToken(String user) throws IOException, InterruptedException,
+ TException {
+ final HiveConf c = new HiveConf();
+ LOG.debug("Creating hive metastore delegation token for user " + user);
+ final UserGroupInformation ugi = UgiFactory.getUgi(user);
+ UserGroupInformation real = ugi.getRealUser();
+ return real.doAs(new PrivilegedExceptionAction<String>() {
+ public String run() throws IOException, TException, InterruptedException {
+ final HiveMetaStoreClient client = new HiveMetaStoreClient(c);
+ return ugi.doAs(new PrivilegedExceptionAction<String>() {
+ public String run() throws IOException, TException, InterruptedException {
+ String u = ugi.getUserName();
+ return client.getDelegationToken(u);
+ }
+ });
+ }
+ });
}
}
Modified: hive/branches/maven/hwi/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/maven/hwi/ivy.xml?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hwi/ivy.xml (original)
+++ hive/branches/maven/hwi/ivy.xml Tue Oct 22 17:58:59 2013
@@ -29,6 +29,8 @@
<dependency org="org.apache.hive" name="hive-cli" rev="${version}"
conf="compile->default" />
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" />
+ <dependency org="org.apache.ant" name="ant" rev="${apacheant.version}" />
+ <dependency org="org.apache.ant" name="ant-launcher" rev="${apacheant.version}" />
<!-- Test Dependencies -->
<dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
Modified: hive/branches/maven/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java (original)
+++ hive/branches/maven/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java Tue Oct 22 17:58:59 2013
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
+import java.sql.SQLException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -332,8 +333,12 @@ public class HWISessionItem implements R
String cmd_trimmed = cmd.trim();
String[] tokens = cmd_trimmed.split("\\s+");
String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim();
-
- CommandProcessor proc = CommandProcessorFactory.get(tokens[0]);
+ CommandProcessor proc = null;
+ try {
+ proc = CommandProcessorFactory.get(tokens[0]);
+ } catch (SQLException e) {
+ l4j.error(getSessionName() + " error processing " + cmd, e);
+ }
if (proc != null) {
if (proc instanceof Driver) {
Driver qp = (Driver) proc;
Modified: hive/branches/maven/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/maven/ivy/libraries.properties?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/ivy/libraries.properties (original)
+++ hive/branches/maven/ivy/libraries.properties Tue Oct 22 17:58:59 2013
@@ -17,7 +17,7 @@
# project configuration files.
# These are the versions of our dependencies (in alphabetical order)
-apacheant.version=1.7.1
+apacheant.version=1.8.4
ant-contrib.version=1.0b3
ant-task.version=2.0.10
antlr.version=3.4
@@ -37,11 +37,12 @@ commons-configuration.version=1.6
commons-httpclient.version=3.0.1
commons-io.version=2.4
commons-lang.version=2.4
-commons-logging.version=1.0.4
+commons-logging.version=1.1.1
commons-logging-api.version=1.0.4
commons-pool.version=1.5.4
derby.version=10.4.2.0
guava.version=11.0.2
+groovy.version=2.1.6
hbase.version=0.94.6.1
httpclient.version=4.2.5
httpcore.version=4.2.4
Modified: hive/branches/maven/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/maven/metastore/ivy.xml?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/metastore/ivy.xml (original)
+++ hive/branches/maven/metastore/ivy.xml Tue Oct 22 17:58:59 2013
@@ -31,7 +31,10 @@
<dependency org="org.antlr" name="ST4" rev="${ST4.version}" transitive="false"/><!-- manually added (antlr dep), bad POM -->
<dependency org="org.apache.hive" name="hive-serde" rev="${version}"
conf="compile->default" />
- <dependency org="com.jolbox" name="bonecp" rev="${BoneCP.version}"/>
+ <dependency org="com.jolbox" name="bonecp" rev="${BoneCP.version}">
+ <exclude org="com.google.guava" module="guava"/>
+ </dependency>
+
<dependency org="commons-pool" name="commons-pool" rev="${commons-pool.version}"/>
<dependency org="org.datanucleus" name="datanucleus-api-jdo" rev="${datanucleus-api-jdo.version}">
<exclude org="javax.jdo" module="jdo2-api"/>
Modified: hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Tue Oct 22 17:58:59 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.model.MDatabase;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
+import org.apache.hadoop.hive.serde.serdeConstants;
/**
* This class contains the optimizations for MetaStore that rely on direct SQL access to
@@ -101,7 +103,9 @@ class MetaStoreDirectSql {
tx = pm.currentTransaction();
tx.begin();
}
- // This should work. If it doesn't, we will self-disable. What a PITA...
+ // Force the underlying db to initialize.
+ pm.newQuery(MDatabase.class, "name == ''").execute();
+ // Self-test query. If it doesn't work, we will self-disable. What a PITA...
boolean isCompatibleDatastore = false;
String selfTestQuery = "select \"DB_ID\" from \"DBS\"";
try {
@@ -165,7 +169,7 @@ class MetaStoreDirectSql {
}
String list = repeat(",?", partNames.size()).substring(1);
return getPartitionsViaSqlFilterInternal(dbName, tblName, null,
- "and \"PARTITIONS\".\"PART_NAME\" in (" + list + ")",
+ "\"PARTITIONS\".\"PART_NAME\" in (" + list + ")",
partNames, new ArrayList<String>(), max);
}
@@ -179,7 +183,8 @@ class MetaStoreDirectSql {
public List<Partition> getPartitionsViaSqlFilter(
Table table, ExpressionTree tree, Integer max) throws MetaException {
assert tree != null;
- List<String> params = new ArrayList<String>(), joins = new ArrayList<String>();
+ List<Object> params = new ArrayList<Object>();
+ List<String> joins = new ArrayList<String>();
String sqlFilter = PartitionFilterGenerator.generateSqlFilter(table, tree, params, joins);
if (sqlFilter == null) {
return null; // Cannot make SQL filter to push down.
@@ -232,7 +237,7 @@ class MetaStoreDirectSql {
* @return List of partition objects.
*/
private List<Partition> getPartitionsViaSqlFilterInternal(String dbName, String tblName,
- Boolean isView, String sqlFilter, List<String> paramsForFilter,
+ Boolean isView, String sqlFilter, List<? extends Object> paramsForFilter,
List<String> joinsForFilter, Integer max) throws MetaException {
boolean doTrace = LOG.isDebugEnabled();
dbName = dbName.toLowerCase();
@@ -255,9 +260,11 @@ class MetaStoreDirectSql {
String queryText =
"select \"PARTITIONS\".\"PART_ID\" from \"PARTITIONS\""
+ " inner join \"TBLS\" on \"PARTITIONS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\" "
+ + " and \"TBLS\".\"TBL_NAME\" = ? "
+ " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" "
- + join(joinsForFilter, ' ') + " where \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? "
- + (sqlFilter == null ? "" : sqlFilter) + orderForFilter;
+ + " and \"DBS\".\"NAME\" = ? "
+ + join(joinsForFilter, ' ')
+ + (sqlFilter == null ? "" : (" where " + sqlFilter)) + orderForFilter;
Object[] params = new Object[paramsForFilter.size() + 2];
params[0] = tblName;
params[1] = dbName;
@@ -649,11 +656,11 @@ class MetaStoreDirectSql {
private static class PartitionFilterGenerator extends TreeVisitor {
private final Table table;
private final FilterBuilder filterBuffer;
- private final List<String> params;
+ private final List<Object> params;
private final List<String> joins;
private PartitionFilterGenerator(
- Table table, List<String> params, List<String> joins) {
+ Table table, List<Object> params, List<String> joins) {
this.table = table;
this.params = params;
this.joins = joins;
@@ -668,7 +675,7 @@ class MetaStoreDirectSql {
* @return the string representation of the expression tree
*/
public static String generateSqlFilter(Table table,
- ExpressionTree tree, List<String> params, List<String> joins) throws MetaException {
+ ExpressionTree tree, List<Object> params, List<String> joins) throws MetaException {
assert table != null;
if (tree.getRoot() == null) {
return "";
@@ -685,7 +692,7 @@ class MetaStoreDirectSql {
if (joins.get(i) != null) continue;
joins.remove(i--);
}
- return "and (" + visitor.filterBuffer.getFilter() + ")";
+ return "(" + visitor.filterBuffer.getFilter() + ")";
}
@Override
@@ -718,10 +725,28 @@ class MetaStoreDirectSql {
int partColIndex = node.getPartColIndexForFilter(table, filterBuffer);
if (filterBuffer.hasError()) return;
- // Add parameters linearly; we are traversing leaf nodes LTR, so they would match correctly.
- String valueAsString = node.getFilterPushdownParam(table, partColIndex, filterBuffer);
- if (filterBuffer.hasError()) return;
- params.add(valueAsString);
+ // We skipped 'like', other ops should all work as long as the types are right.
+ String colType = table.getPartitionKeys().get(partColIndex).getType();
+ boolean isStringCol = colType.equals(serdeConstants.STRING_TYPE_NAME);
+ if (!isStringCol && !serdeConstants.IntegralTypes.contains(colType)) {
+ filterBuffer.setError("Filter pushdown is only supported for string or integral columns");
+ return;
+ }
+
+ boolean isStringVal = node.value instanceof String;
+ if (!isStringVal && !(node.value instanceof Long)) {
+ filterBuffer.setError("Filter pushdown is only supported for string or integral values");
+ return;
+ } else if (isStringCol != isStringVal) {
+ // It's not clear how filtering for e.g. "stringCol > 5" should work (which side is
+ // to be coerced?). Let the expression evaluation sort this one out, not metastore.
+ filterBuffer.setError("Cannot push down filter for "
+ + (isStringCol ? "string" : "integral") + " column and value " + node.value);
+ return;
+ }
+
+ // Force string-based handling in some cases to be compatible with JDO pushdown.
+ boolean forceStringEq = !isStringCol && node.canJdoUseStringsWithIntegral();
if (joins.isEmpty()) {
// There's a fixed number of partition cols that we might have filters on. To avoid
@@ -738,8 +763,19 @@ class MetaStoreDirectSql {
+ " and \"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex);
}
+ // Build the filter and add parameters linearly; we are traversing leaf nodes LTR.
String tableValue = "\"FILTER" + partColIndex + "\".\"PART_KEY_VAL\"";
- // TODO: need casts here if #doesOperatorSupportIntegral is amended to include lt/gt/etc.
+ if (!isStringCol && !forceStringEq) {
+ // The underlying database field is varchar, we need to compare numbers.
+ tableValue = "cast(" + tableValue + " as decimal(21,0))";
+ // This is a workaround for DERBY-6358; as such, it is pretty horrible.
+ tableValue = "(case when \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? then "
+ + tableValue + " else null end)";
+ params.add(table.getTableName().toLowerCase());
+ params.add(table.getDbName().toLowerCase());
+ }
+ params.add(forceStringEq ? node.value.toString() : node.value);
+
filterBuffer.append(node.isReverseOrder
? "(? " + node.operator.getSqlOp() + " " + tableValue + ")"
: "(" + tableValue + " " + node.operator.getSqlOp() + " ?)");
Modified: hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java?rev=1534711&r1=1534710&r2=1534711&view=diff
==============================================================================
--- hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java (original)
+++ hive/branches/maven/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java Tue Oct 22 17:58:59 2013
@@ -130,9 +130,9 @@ public class MetaStoreSchemaInfo {
return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION;
}
- // Current hive version, remove the 'SNAPSHOT' part if needed
+ // Current hive version, in majorVersion.minorVersion.changeVersion format
public static String getHiveSchemaVersion() {
- return HiveVersionInfo.getVersion().replace("-SNAPSHOT", "");
+ return HiveVersionInfo.getShortVersion();
}
}