You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/07/29 23:08:19 UTC
svn commit: r1508202 [3/48] - in /hive/branches/tez: ./
beeline/src/java/org/apache/hive/beeline/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metrics/
common/src/java/org/apache/hadoop/hive/conf/ common/src/te...
Modified: hive/branches/tez/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/build.xml?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/build.xml (original)
+++ hive/branches/tez/hcatalog/build.xml Mon Jul 29 21:08:03 2013
@@ -26,7 +26,10 @@
<!-- <import file="../build-common.xml"/> -->
<property name="path.to.basedir" location="${basedir}"/>
-
+ <property name="test.result.dir" location="${build.dir.hive}/hcatalog/test"
+ description="location to place TEST-*.xml files so that hive's testreport includes them"/>
+ <property name="hive.dist.dir" location="${build.dir.hive}/dist"
+ description="where all hive build artifacts are placed to make binary tar ball"/>
<loadproperties srcfile="${basedir}/build.properties"/>
<!--
@@ -91,16 +94,31 @@
<target name="gen-test" description="Generate tests, a no-op for hcat"/>
<target name="test" depends="jar" description="run unit tests">
+ <mkdir dir="${test.result.dir}"/>
<!-- Placed in a parallel structure so that the tests keep going
even if some fail. Otherwise a failure in one of the earlier ant
call terminates the target and the rest do not run. -->
<parallel threadCount="1">
- <ant target="test" dir="core" inheritAll="false"/>
- <ant target="test" dir="hcatalog-pig-adapter" inheritAll="false"/>
- <ant target="test" dir="server-extensions" inheritAll="false"/>
- <ant target="test" dir="webhcat/svr" inheritAll="false"/>
- <ant target="test" dir="webhcat/java-client" inheritAll="false"/>
- <ant target="test" dir="storage-handlers/hbase" inheritAll="false"/>
+ <ant target="test" dir="core" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ </ant>
+ <ant target="test" dir="hcatalog-pig-adapter" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ </ant>
+ <ant target="test" dir="server-extensions" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ </ant>
+ <ant target="test" dir="webhcat/svr" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ <property name="env.HIVE_HOME" value="${hive.dist.dir}"/>
+ <property name="env.HCAT_PREFIX" value="${hive.dist.dir}/hcatalog"/>
+ </ant>
+ <ant target="test" dir="webhcat/java-client" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ </ant>
+ <ant target="test" dir="storage-handlers/hbase" inheritAll="false">
+ <property name="test.result.dir" location="${test.result.dir}"/>
+ </ant>
<!-- One checkstyle run for the whole repo. Runs after junit tests
to piggyback on resolved jars. -->
<path id="checkstyle.class.path">
@@ -174,6 +192,7 @@
<!-- Clean up children -->
<target name="clean-test" description="Cleanup test artifacts">
<echo message="${ant.project.name}"/>
+ <delete dir="${test.result.dir}"/>
<delete dir="${build.dir}"/>
<ant target="clean-test" dir="core" inheritAll="false"/>
<ant target="clean-test" dir="hcatalog-pig-adapter" inheritAll="false"/>
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/HCatRecordSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/HCatRecordSerDe.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/HCatRecordSerDe.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/HCatRecordSerDe.java Mon Jul 29 21:08:03 2013
@@ -266,28 +266,29 @@ public class HCatRecordSerDe implements
private static Object serializePrimitiveField(Object field,
ObjectInspector fieldObjectInspector) {
- if (field != null && HCatContext.INSTANCE.getConf().isPresent()) {
+ Object f = ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
+ if (f != null && HCatContext.INSTANCE.getConf().isPresent()) {
Configuration conf = HCatContext.INSTANCE.getConf().get();
- if (field instanceof Boolean &&
+ if (f instanceof Boolean &&
conf.getBoolean(
HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) {
- return ((Boolean) field) ? 1 : 0;
- } else if (field instanceof Short &&
+ return ((Boolean) f) ? 1 : 0;
+ } else if (f instanceof Short &&
conf.getBoolean(
HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
- return new Integer((Short) field);
- } else if (field instanceof Byte &&
+ return new Integer((Short) f);
+ } else if (f instanceof Byte &&
conf.getBoolean(
HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) {
- return new Integer((Byte) field);
+ return new Integer((Byte) f);
}
}
- return ((PrimitiveObjectInspector) fieldObjectInspector).getPrimitiveJavaObject(field);
+ return f;
}
/**
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/data/schema/HCatSchemaUtils.java Mon Jul 29 21:08:03 2013
@@ -31,8 +31,6 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hcatalog.common.HCatConstants;
-import org.apache.hcatalog.common.HCatContext;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.data.schema.HCatFieldSchema.Type;
@@ -140,17 +138,9 @@ public class HCatSchemaUtils {
private static Type getPrimitiveHType(TypeInfo basePrimitiveTypeInfo) {
switch (((PrimitiveTypeInfo) basePrimitiveTypeInfo).getPrimitiveCategory()) {
case BOOLEAN:
- return (HCatContext.INSTANCE.getConf().isPresent() &&
- HCatContext.INSTANCE.getConf().get().getBoolean(
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER,
- HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER_DEFAULT)) ?
- Type.INT : Type.BOOLEAN;
+ return Type.BOOLEAN;
case BYTE:
- return (HCatContext.INSTANCE.getConf().isPresent() &&
- HCatContext.INSTANCE.getConf().get().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
- Type.INT : Type.TINYINT;
+ return Type.TINYINT;
case DOUBLE:
return Type.DOUBLE;
case FLOAT:
@@ -160,11 +150,7 @@ public class HCatSchemaUtils {
case LONG:
return Type.BIGINT;
case SHORT:
- return (HCatContext.INSTANCE.getConf().isPresent() &&
- HCatContext.INSTANCE.getConf().get().getBoolean(
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION,
- HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION_DEFAULT)) ?
- Type.INT : Type.SMALLINT;
+ return Type.SMALLINT;
case STRING:
return Type.STRING;
case BINARY:
Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java Mon Jul 29 21:08:03 2013
@@ -101,4 +101,6 @@ public interface HCatHadoopShims {
*/
public boolean isFileInHDFS(FileSystem fs, Path path) throws IOException;
+ public InetSocketAddress getAddress(Configuration conf);
+
}
Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java Mon Jul 29 21:08:03 2013
@@ -104,6 +104,10 @@ class PigHCatUtil {
}
}
+ static public boolean pigHasBooleanSupport(){
+ return pigHasBooleanSupport;
+ }
+
static public Pair<String, String> getDBTableNames(String location) throws IOException {
// the location string will be of the form:
// <database name>.<table name> - parse it and
Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java Mon Jul 29 21:08:03 2013
@@ -65,7 +65,7 @@ public class TestHCatLoader extends Test
private static int guardTestCount = 6; // ugh, instantiate using introspection in guardedSetupBeforeClass
private static boolean setupHasRun = false;
-
+
private static Map<Integer, Pair<Integer, String>> basicInputData;
protected String storageFormat() {
@@ -413,7 +413,7 @@ public class TestHCatLoader extends Test
File inputDataDir = new File(inputFileName).getParentFile();
inputDataDir.mkdir();
- String[] lines = new String[]{"llama\t1", "alpaca\t0"};
+ String[] lines = new String[]{"llama\ttrue", "alpaca\tfalse"};
HcatTestUtils.createTestDataFile(inputFileName, lines);
assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode());
@@ -433,13 +433,15 @@ public class TestHCatLoader extends Test
assertEquals("a", schema.getField(0).alias);
assertEquals(DataType.CHARARRAY, schema.getField(0).type);
assertEquals("b", schema.getField(1).alias);
- assertEquals(DataType.INTEGER, schema.getField(1).type);
+ if (PigHCatUtil.pigHasBooleanSupport()){
+ assertEquals(DataType.BOOLEAN, schema.getField(1).type);
+ } else {
+ assertEquals(DataType.INTEGER, schema.getField(1).type);
+ }
Iterator<Tuple> iterator = server.openIterator("data");
Tuple t = iterator.next();
assertEquals("llama", t.get(0));
- // TODO: Figure out how to load a text file into Hive with boolean columns. This next assert
- // passes because data was loaded as integers, not because it was converted.
assertEquals(1, t.get(1));
t = iterator.next();
assertEquals("alpaca", t.get(0));
Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java Mon Jul 29 21:08:03 2013
@@ -60,14 +60,18 @@ public class TestHCatLoaderComplexSchema
driver.run("drop table " + tablename);
}
+ protected String storageFormat() {
+ return "RCFILE tblproperties('hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver'," +
+ "'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver')";
+ }
+
private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
String createTable;
createTable = "create table " + tablename + "(" + schema + ") ";
if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) {
createTable = createTable + "partitioned by (" + partitionedBy + ") ";
}
- createTable = createTable + "stored as RCFILE tblproperties('hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver'," +
- "'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver') ";
+ createTable = createTable + "stored as " + storageFormat();
LOG.info("Creating table:\n {}", createTable);
CommandProcessorResponse result = driver.run(createTable);
int retCode = result.getResponseCode();
Modified: hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original)
+++ hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 29 21:08:03 2013
@@ -19,12 +19,12 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.shims.HCatHadoopShims;
/*
* Communicate with the JobTracker as a specific user.
@@ -35,14 +35,13 @@ public class TempletonJobTracker {
/**
* Create a connection to the Job Tracker.
*/
- public TempletonJobTracker(InetSocketAddress addr,
- Configuration conf)
+ public TempletonJobTracker(Configuration conf)
throws IOException {
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
cnx = (JobSubmissionProtocol)
RPC.getProxy(JobSubmissionProtocol.class,
JobSubmissionProtocol.versionID,
- addr,
+ HCatHadoopShims.Instance.get().getAddress(conf),
ugi,
conf,
NetUtils.getSocketFactory(conf,
Modified: hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (original)
+++ hive/branches/tez/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Mon Jul 29 21:08:03 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Outpu
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progressable;
public class HCatHadoopShims20S implements HCatHadoopShims {
@@ -149,4 +150,11 @@ public class HCatHadoopShims20S implemen
// In hadoop 1.x.x the file system URI is sufficient to determine the uri of the file
return "hdfs".equals(fs.getUri().getScheme());
}
+
+ @Override
+ public InetSocketAddress getAddress(Configuration conf) {
+ String jobTrackerStr =
+ conf.get("mapred.job.tracker", "localhost:8012");
+ return NetUtils.createSocketAddr(jobTrackerStr);
+ }
}
Modified: hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original)
+++ hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 29 21:08:03 2013
@@ -19,8 +19,6 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.InetSocketAddress;
-
import org.apache.hadoop.conf.Configuration;
/*
@@ -32,8 +30,7 @@ public class TempletonJobTracker {
/**
* Create a connection to the Job Tracker.
*/
- public TempletonJobTracker(InetSocketAddress addr,
- Configuration conf)
+ public TempletonJobTracker(Configuration conf)
throws IOException {
jc = new JobClient(conf);
Modified: hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original)
+++ hive/branches/tez/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Mon Jul 29 21:08:03 2013
@@ -127,4 +127,9 @@ public class HCatHadoopShims23 implement
// resolvePath is a sure shot way of knowing which file system the file is.
return "hdfs".equals(fs.resolvePath(path).toUri().getScheme());
}
+
+ @Override
+ public InetSocketAddress getAddress(Configuration conf) {
+ return null;
+ }
}
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/build.xml?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/build.xml (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/build.xml Mon Jul 29 21:08:03 2013
@@ -262,12 +262,21 @@
<target name="test" depends="property-check, udfs, tar, init-test">
+ <!-- fork (parallelization) factors for e2e tests execution.
+ Defaults are 1, which means *no* parellelization: -->
+ <property name="fork.factor.group" value="1"/>
+ <property name="fork.factor.conf.file" value="1"/>
+ <property name="e2e.debug" value="false"/>
+
<!-- If they have not specified tests to run then null it out -->
<property name="tests.to.run" value=""/>
<echo/>
<exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="."/>
<env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
+ <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
+ <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
+ <env key="E2E_DEBUG" value="${e2e.debug}"/>
<env key="HADOOP_HOME" value="${hadoop.home}"/>
<env key="HADOOP_CONF_DIR" value="${harness.cluster.conf}"/>
<env key="HIVE_HOME" value="${hive.home}"/>
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm Mon Jul 29 21:08:03 2013
@@ -59,13 +59,13 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
- my $subName = (caller(0))[3];
-
# Setup the output path
my $me = `whoami`;
chomp $me;
- $globalHash->{'runid'} = $me . "." . time;
+ my $jobId = $globalHash->{'job-id'};
+ my $timeId = time;
+ $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
# if "-ignore false" was provided on the command line,
# it means do run tests even when marked as 'ignore'
@@ -76,6 +76,12 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
+}
+
+sub globalSetupConditional
+{
+ my ($self, $globalHash, $log) = @_;
# add libexec location to the path
if (defined($ENV{'PATH'})) {
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Mon Jul 29 21:08:03 2013
@@ -56,13 +56,13 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
- my $subName = (caller(0))[3];
-
# Setup the output path
my $me = `whoami`;
chomp $me;
- $globalHash->{'runid'} = $me . "." . time;
+ my $jobId = $globalHash->{'job-id'};
+ my $timeId = time;
+ $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
# if "-ignore false" was provided on the command line,
# it means do run tests even when marked as 'ignore'
@@ -73,6 +73,11 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
+}
+
+sub globalSetupConditional() {
+ my ($self, $globalHash, $log) = @_;
# add libexec location to the path
if (defined($ENV{'PATH'})) {
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHive.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHive.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHive.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverHive.pm Mon Jul 29 21:08:03 2013
@@ -57,14 +57,16 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
- my $subName = (caller(0))[3];
# Setup the output path
my $me = `whoami`;
chomp $me;
- $globalHash->{'runid'} = $me . "." . time;
+ my $jobId = $globalHash->{'job-id'};
+ my $timeId = time;
+ $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
die "Cannot create localpath directory " . $globalHash->{'localpath'} .
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Mon Jul 29 21:08:03 2013
@@ -56,13 +56,13 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
- my $subName = (caller(0))[3];
-
# Setup the output path
my $me = `whoami`;
chomp $me;
- $globalHash->{'runid'} = $me . "." . time;
+ my $jobId = $globalHash->{'job-id'};
+ my $timeId = time;
+ $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
# if "-ignore false" was provided on the command line,
# it means do run tests even when marked as 'ignore'
@@ -73,6 +73,12 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
+}
+
+sub globalSetupConditional
+{
+ my ($self, $globalHash, $log) = @_;
# add libexec location to the path
if (defined($ENV{'PATH'})) {
@@ -109,6 +115,12 @@ sub globalSetup
sub globalCleanup
{
+ # noop there because the removal of temp directories, which are created in #globalSetupConditional(), is to be
+ # performed in method #globalCleanupConditional().
+}
+
+sub globalCleanupConditional
+{
my ($self, $globalHash, $log) = @_;
IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
Modified: hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/hcatalog/drivers/Util.pm Mon Jul 29 21:08:03 2013
@@ -421,10 +421,10 @@ sub getHiveLibsForPig($$)
/datanucleus-.*\.jar$/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
- /jdo2-api-.*\.jar$/ && do {
+ /jdo-api-.*\.jar$/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
- /commons-dbcp-.*\.jar$/ && do {
+ /bonecp-.*\.jar$/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
/commons-pool-.*\.jar$/ && do {
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/README.txt?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/README.txt (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/README.txt Mon Jul 29 21:08:03 2013
@@ -150,3 +150,14 @@ Also useful to add to conf/hadoop-env.sh
export HADOOP_OPTS="-Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk"
to prevent warning about SCDynamicStore which may throw some tests off
(http://stackoverflow.com/questions/7134723/hadoop-on-osx-unable-to-load-realm-info-from-scdynamicstore)
+
+
+Performance
+-----------
+It's a good idea to set fork.factor.conf.file={number of .conf files} and fork.factor.group to something > 1
+(see build.xml) to make these tests run faster. If doing this, make sure the Hadoop Cluster has
+enough map slots (10?) (mapred.tasktracker.map.tasks.maximum), otherwise test parallelism won't help.
+
+Adding Tests
+------------
+ToDo: add some guidelines
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/build.xml?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/build.xml Mon Jul 29 21:08:03 2013
@@ -82,6 +82,11 @@
</target>
<target name="test" depends="property-check, init-test">
+ <!-- fork (parallelization) factors for e2e tests execution.
+ Defaults are 1, which means *no* parellelization: -->
+ <property name="fork.factor.group" value="1"/>
+ <property name="fork.factor.conf.file" value="1"/>
+ <property name="e2e.debug" value="false"/>
<property name="tests.to.run" value=""/>
<exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="."/>
@@ -90,6 +95,9 @@
<env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
<env key="TH_OUT" value="."/>
<env key="TH_ROOT" value="."/>
+ <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
+ <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
+ <env key="E2E_DEBUG" value="${e2e.debug}"/>
<env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
<env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
<env key="USER_NAME" value="${test.user.name}"/>
@@ -99,10 +107,16 @@
<arg value="${basedir}/tests/serverstatus.conf"/>
<arg value="${basedir}/tests/ddl.conf"/>
<arg value="${basedir}/tests/jobsubmission.conf"/>
+ <arg value="${basedir}/tests/jobsubmission2.conf"/>
</exec>
</target>
<target name="test-hcat-authorization" depends="property-check, init-test">
+ <!-- fork (parallelization) factors for e2e tests execution.
+ Defaults are 1, which means *no* parellelization: -->
+ <property name="fork.factor.group" value="1"/>
+ <property name="fork.factor.conf.file" value="1"/>
+ <property name="e2e.debug" value="false"/>
<property name="tests.to.run" value=""/>
<exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="${harness.dir}"/>
@@ -111,6 +125,9 @@
<env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
<env key="TH_OUT" value="."/>
<env key="TH_ROOT" value="."/>
+ <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
+ <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
+ <env key="E2E_DEBUG" value="${e2e.debug}"/>
<env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
<env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
<env key="USER_NAME" value="${test.user.name}"/>
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Mon Jul 29 21:08:03 2013
@@ -150,13 +150,13 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
- my $subName = (caller(0))[3];
-
# Setup the output path
my $me = `whoami`;
chomp $me;
- $globalHash->{'runid'} = $me . "." . time;
+ my $jobId = $globalHash->{'job-id'};
+ my $timeId = time;
+ $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
# if "-ignore false" was provided on the command line,
# it means do run tests even when marked as 'ignore'
@@ -170,6 +170,7 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'};
$globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'};
$globalHash->{'current_user'} = $ENV{'USER_NAME'};
@@ -184,6 +185,11 @@ sub globalSetup
$globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'};
$globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'};
+ }
+
+sub globalSetupConditional
+ {
+ my ($self, $globalHash, $log) = @_;
# add libexec location to the path
if (defined($ENV{'PATH'})) {
@@ -218,6 +224,12 @@ sub globalSetup
# None
sub globalCleanup
{
+ # noop there because the removal of temp directories, which are created in #globalSetupConditional(), is to be
+ # performed in method #globalCleanupConditional().
+ }
+
+sub globalCleanupConditional
+ {
my ($self, $globalHash, $log) = @_;
IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
@@ -669,7 +681,12 @@ sub compare
#try to get the call back url request until timeout
if ($result == 1 && defined $testCmd->{'check_call_back'}) {
my $d = $testCmd->{'http_daemon'};
- $d->timeout(300); #wait for 5 mins
+ if (defined $testCmd->{'timeout_seconds'}) {
+ $d->timeout($testCmd->{'timeout_seconds'})
+ }
+ else {
+ $d->timeout(300); #wait for 5 mins by default
+ }
my $url_requested;
$testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g;
print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n";
@@ -720,6 +737,10 @@ sub compare
my $jobComplete;
my $NUM_RETRIES = 60;
my $SLEEP_BETWEEN_RETRIES = 5;
+ if (defined $testCmd->{'timeout_seconds'} && $testCmd->{'timeout_seconds'} > 0) {
+ $SLEEP_BETWEEN_RETRIES = ($testCmd->{'timeout_seconds'} / $NUM_RETRIES);
+ print $log "found timeout_seconds & set SLEEP_BETWEEN_RETRIES=$SLEEP_BETWEEN_RETRIES";
+ }
#first wait for job completion
while ($NUM_RETRIES-- > 0) {
Modified: hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/ddl.conf
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/ddl.conf?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/ddl.conf (original)
+++ hive/branches/tez/hcatalog/src/test/e2e/templeton/tests/ddl.conf Mon Jul 29 21:08:03 2013
@@ -283,8 +283,8 @@ $cfg =
'status_code' => 200,
# 'json_field_substr_match' => {'table-name' => 'templeton_testtab1'},
'json_field_match_object' => { 'columns' => '[
- { "name" : "i", "type" : "int"},
- { "name" : "j", "type" : "bigint", "comment" : "some comment" }
+ { "name" : "i", "type" : "int", "comment" : "from deserializer" },
+ { "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
]' },
},
{ #drop table
@@ -299,7 +299,7 @@ $cfg =
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:',
'status_code' => 404,
- 'json_field_substr_match' => {'error' => 'templeton_testtab1 does not exist'},
+ 'json_field_substr_match' => {'error' => 'FAILED: SemanticException \[Error 10001\]: Table not found templeton_testtab1'},
},
{
#-ve test , describe on non existent table
@@ -307,7 +307,7 @@ $cfg =
'method' => 'GET',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl/database/templeton_testdb1/table/templeton_testtab1?user.name=:UNAME:&format=extended:',
'status_code' => 404,
- 'json_field_substr_match' => {'error' => 'templeton_testtab1 does not exist'},
+ 'json_field_substr_match' => {'error' => 'FAILED: SemanticException \[Error 10001\]: Table not found templeton_testtab1'},
},
]
@@ -399,16 +399,15 @@ $cfg =
{'columns' => '[
{
"name" : "i",
- "type" : "int"
+ "type" : "int", "comment" : "from deserializer"
},
{
"name" : "j",
- "type" : "bigint"
+ "type" : "bigint", "comment" : "from deserializer"
},
{
- "comment" : "IP Address of the User",
"name" : "ip",
- "type" : "string"
+ "type" : "string", "comment" : "from deserializer"
}
]',
"partitionColumns" => '[
@@ -480,8 +479,8 @@ $cfg =
'status_code' => 200,
# 'json_field_substr_match' => {'table-name' => 'templeton_testtab1'},
'json_field_match_object' => { 'columns' => '[
- { "name" : "i", "type" : "int" },
- { "name" : "j", "type" : "bigint" }
+ { "name" : "i", "type" : "int", "comment" : "from deserializer" },
+ { "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
]' },
},
@@ -626,16 +625,15 @@ STORED AS rcfile
"columns" => '[
{
"name" : "i",
- "type" : "int"
+ "type" : "int", "comment" : "from deserializer"
},
{
"name" : "j",
- "type" : "bigint"
+ "type" : "bigint", "comment" : "from deserializer"
},
{
- "comment" : "IP Address of the User",
"name" : "ip",
- "type" : "string"
+ "type" : "string", "comment" : "from deserializer"
}
]',
@@ -679,16 +677,15 @@ STORED AS rcfile
"columns" => '[
{
"name" : "i",
- "type" : "int"
+ "type" : "int", "comment" : "from deserializer"
},
{
"name" : "j",
- "type" : "bigint"
+ "type" : "bigint", "comment" : "from deserializer"
},
{
- "comment" : "IP Address of the User",
"name" : "ip",
- "type" : "string"
+ "type" : "string", "comment" : "from deserializer"
}
]',
@@ -735,7 +732,7 @@ STORED AS rcfile
'status_code' => 404,
'json_field_substr_match' =>
{
- 'error' => 'Partition.*for table templetontest_parts does not exist'
+ 'error' => 'FAILED: SemanticException \[Error 10006\]: Partition not found \{dt=20120101\, country=IN\}'
},
},
@@ -782,8 +779,8 @@ STORED AS rcfile
'json_field_match_object' =>
{
'columns' => '[
- { "name" : "i", "type" : "int", "comment" : "column with comment"},
- { "name" : "j", "type" : "bigint" }
+ { "name" : "i", "type" : "int", "comment" : "from deserializer"},
+ { "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
]'
},
},
@@ -802,7 +799,7 @@ STORED AS rcfile
},
'json_field_match_object' =>
{
- 'column' => '{ "name" : "i", "type" : "int", "comment" : "column with comment" }'
+ 'column' => '{ "name" : "i", "type" : "int", "comment" : "from deserializer" }'
},
},
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/build.xml?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/build.xml (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/build.xml Mon Jul 29 21:08:03 2013
@@ -63,7 +63,6 @@
<property name="test.tmp.dir" value="${test.build.dir}/temp" />
<property name="test.data.dir" value="${test.build.dir}/data" />
<property name="test.timeout" value="2700000" />
- <property name="test.junit.output.format" value="plain" />
<property name="test.output" value="no"/>
<property name="hive.conf.dir" value="${hive.root}/conf"/>
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/bin/webhcat_config.sh Mon Jul 29 21:08:03 2013
@@ -18,6 +18,12 @@
# under the License.
+# Print an error message and exit
+function die() {
+ echo "${this}: $@" 1>&2
+ exit 1
+}
+
#====================================
#Default config param values
#====================================
@@ -46,7 +52,7 @@ SLEEP_TIME_AFTER_START=10
#================================================
#These parameters can be overriden by webhcat-env.sh
-# the root of the WEBHCAT installation
+# the root of the WEBHCAT installation ('this' is defined in webhcat_server.sh)
export WEBHCAT_PREFIX=`dirname "$this"`/..
#check to see if the conf dir is given as an optional argument
@@ -62,15 +68,32 @@ then
fi
# Allow alternate conf dir location.
-if [ -e "${WEBHCAT_PREFIX}/etc/webhcat/webhcat-env.sh" ]; then
+if [ -e "${WEBHCAT_PREFIX}/etc/webhcat/webhcat-env.sh" -o -e "${WEBHCAT_PREFIX}/etc/webhcat/webhcat-site.xml" ]; then
DEFAULT_CONF_DIR=${WEBHCAT_PREFIX}/"etc/webhcat"
-elif [ -e "${WEBHCAT_PREFIX}/conf/webhcat-env.sh" ]; then
+elif [ -e "${WEBHCAT_PREFIX}/conf/webhcat-env.sh" -o -e "${WEBHCAT_PREFIX}/etc/webhcat/webhcat-site.xml" ]; then
DEFAULT_CONF_DIR=${WEBHCAT_PREFIX}/"conf"
else
DEFAULT_CONF_DIR="/etc/webhcat"
fi
WEBHCAT_CONF_DIR="${WEBHCAT_CONF_DIR:-$DEFAULT_CONF_DIR}"
+#set defaults for HCAT_PREFIX, HIVE_HOME, TEMPLETON_HOME that work for default directory structure
+DEFAULT_HCAT_PREFIX="${WEBHCAT_PREFIX}"
+export HCAT_PREFIX="${HCAT_PREFIX:-$DEFAULT_HCAT_PREFIX}"
+if [ ! -f ${HCAT_PREFIX}/bin/hcat ]; then
+ die "HCAT_PREFIX=${HCAT_PREFIX} is invalid";
+fi
+DEFAULT_HIVE_HOME="${WEBHCAT_PREFIX}/.."
+export HIVE_HOME="${HIVE_HOME:-$DEFAULT_HIVE_HOME}"
+if [ ! -f ${HIVE_HOME}/bin/hive ]; then
+ die "HIVE_HOME=${HIVE_HOME} is invalid";
+fi
+DEFAULT_TEMPLETON_HOME="${WEBHCAT_PREFIX}"
+export TEMPLETON_HOME="${TEMPLETON_HOME:-$DEFAULT_TEMPLETON_HOME}"
+if [ ! -d ${TEMPLETON_HOME}/share/webhcat ]; then
+ die "TEMPLETON_HOME=${TEMPLETON_HOME} is invalid";
+fi
+
#users can add various env vars to webhcat-env.sh in the conf
#rather than having to export them before running the command
if [ -f "${WEBHCAT_CONF_DIR}/webhcat-env.sh" ]; then
@@ -89,6 +112,6 @@ elif [ -f ${WEBHCAT_PREFIX}/bin/hadoop ]
HADOOP_PREFIX=$WEBHCAT_PREFIX
#otherwise see if HADOOP_PREFIX is defined
elif [ ! -f ${HADOOP_PREFIX}/bin/hadoop ]; then
- echo "Hadoop not found."
+ echo "${this}: Hadoop not found."
exit 1
fi
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Mon Jul 29 21:08:03 2013
@@ -116,6 +116,7 @@ public class AppConfig extends Configura
public static final String HADOOP_SPECULATIVE_NAME
= "mapred.map.tasks.speculative.execution";
public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
+ public static final String UNIT_TEST_MODE = "templeton.unit.test.mode";
private static final Log LOG = LogFactory.getLog(AppConfig.class);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java Mon Jul 29 21:08:03 2013
@@ -18,11 +18,13 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
/**
* Missing required or badly configured paramater.
*/
public class BadParam extends SimpleWebException {
public BadParam(String msg) {
- super(400, msg);
+ super(HttpStatus.BAD_REQUEST_400, msg);
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java Mon Jul 29 21:08:03 2013
@@ -18,11 +18,13 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
/**
* Simple "we are busy, try again" exception.
*/
public class BusyException extends SimpleWebException {
public BusyException() {
- super(503, "Busy, please retry");
+ super(HttpStatus.SERVICE_UNAVAILABLE_503, "Busy, please retry");
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java Mon Jul 29 21:08:03 2013
@@ -18,11 +18,13 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
/**
* The callback failed when it tried to reach the callback URL.
*/
public class CallbackFailedException extends SimpleWebException {
public CallbackFailedException(String msg) {
- super(400, msg);
+ super(HttpStatus.BAD_REQUEST_400, msg);
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java Mon Jul 29 21:08:03 2013
@@ -24,6 +24,7 @@ import javax.ws.rs.ext.Provider;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.eclipse.jetty.http.HttpStatus;
/**
* Map all exceptions to the Jersey response. This lets us have nice
@@ -36,6 +37,6 @@ public class CatchallExceptionMapper
public Response toResponse(Exception e) {
LOG.error(e.getMessage(), e);
- return SimpleWebException.buildMessage(500, null, e.getMessage());
+ return SimpleWebException.buildMessage(HttpStatus.INTERNAL_SERVER_ERROR_500, null, e.getMessage());
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Mon Jul 29 21:08:03 2013
@@ -39,8 +39,7 @@ public class DeleteDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
throw new BadParam("Invalid jobid: " + id);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatDelegator.java Mon Jul 29 21:08:03 2013
@@ -33,7 +33,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hcatalog.templeton.tool.TempletonUtils;
+import org.eclipse.jetty.http.HttpStatus;
/**
@@ -95,7 +97,15 @@ public class HcatDelegator extends Launc
args.add("-D");
args.add("hive.format=json");
}
-
+ LOG.info("Main.getAppConfigInstance().get(AppConfig.UNIT_TEST_MODE)=" +
+ Main.getAppConfigInstance().get(AppConfig.UNIT_TEST_MODE));
+ if(System.getProperty("hive.metastore.warehouse.dir") != null) {
+ /*when running in unit test mode, pass this property to HCat,
+ which will in turn pass it to Hive to make sure that Hive
+ tries to write to a directory that exists.*/
+ args.add("-D");
+ args.add("hive.metastore.warehouse.dir=" + System.getProperty("hive.metastore.warehouse.dir"));
+ }
return args;
}
@@ -113,12 +123,6 @@ public class HcatDelegator extends Launc
String res = jsonRun(user, exec);
return JsonBuilder.create(res).build();
} catch (HcatException e) {
- if (e.execBean.stderr.contains("SemanticException")) {
- return JsonBuilder.create().
- put("error", "Database " + db + " does not exist")
- .put("errorCode", "404")
- .put("database", db).build();
- }
throw new HcatException("unable to describe database: " + db,
e.execBean, exec);
}
@@ -256,14 +260,6 @@ public class HcatDelegator extends Launc
.put("table", table)
.build();
} catch (HcatException e) {
- if (e.execBean.stderr.contains("SemanticException") &&
- e.execBean.stderr.contains("Table not found")) {
- return JsonBuilder.create().
- put("error", "Table" + db + "." + table + " does not exist" )
- .put("errorCode", "404")
- .put("table", table)
- .put("database", db).build();
- }
throw new HcatException("unable to describe database: " + db,
e.execBean, exec);
}
@@ -470,8 +466,8 @@ public class HcatDelegator extends Launc
return JsonBuilder.mapToJson(tables.get(0));
else {
return JsonBuilder
- .createError(String.format("Table %s does not exist", table),
- JsonBuilder.MISSING).
+ .createError(ErrorMsg.INVALID_TABLE.format(table),
+ ErrorMsg.INVALID_TABLE.getErrorCode()).
buildJson();
}
}
@@ -530,7 +526,7 @@ public class HcatDelegator extends Launc
throws HcatException, NotAuthorizedException, BusyException,
ExecuteException, IOException {
Response res = descTable(user, db, table, true);
- if (res.getStatus() != JsonBuilder.OK)
+ if (res.getStatus() != HttpStatus.OK_200)
return res;
Map props = tableProperties(res.getEntity());
Map found = null;
@@ -556,7 +552,7 @@ public class HcatDelegator extends Launc
throws HcatException, NotAuthorizedException, BusyException,
ExecuteException, IOException {
Response res = descTable(user, db, table, true);
- if (res.getStatus() != JsonBuilder.OK)
+ if (res.getStatus() != HttpStatus.OK_200)
return res;
Map props = tableProperties(res.getEntity());
return JsonBuilder.create()
@@ -645,7 +641,8 @@ public class HcatDelegator extends Launc
+ table + " does not exist" + db + "." + table + " does not exist";
return JsonBuilder.create()
.put("error", emsg)
- .put("errorCode", "404")
+ //this error should really be produced by Hive (DDLTask)
+ .put("errorCode", ErrorMsg.INVALID_PARTITION.getErrorCode())
.put("database", db)
.put("table", table)
.put("partition", partition)
@@ -678,7 +675,8 @@ public class HcatDelegator extends Launc
if (res.indexOf("AlreadyExistsException") > -1) {
return JsonBuilder.create().
put("error", "Partition already exists")
- .put("errorCode", "409")
+ //This error code should really be produced by Hive
+ .put("errorCode", ErrorMsg.PARTITION_EXISTS.getErrorCode())
.put("database", db)
.put("table", table)
.put("partition", desc.partition).build();
@@ -742,13 +740,13 @@ public class HcatDelegator extends Launc
throws SimpleWebException, NotAuthorizedException, BusyException,
ExecuteException, IOException {
Response res = listColumns(user, db, table);
- if (res.getStatus() != JsonBuilder.OK)
+ if (res.getStatus() != HttpStatus.OK_200)
return res;
Object o = res.getEntity();
final Map fields = (o != null && (o instanceof Map)) ? (Map) o : null;
if (fields == null)
- throw new SimpleWebException(500, "Internal error, unable to find column "
+ throw new SimpleWebException(HttpStatus.INTERNAL_SERVER_ERROR_500, "Internal error, unable to find column "
+ column);
@@ -763,7 +761,7 @@ public class HcatDelegator extends Launc
}
}
if (found == null)
- throw new SimpleWebException(500, "unable to find column " + column,
+ throw new SimpleWebException(HttpStatus.INTERNAL_SERVER_ERROR_500, "unable to find column " + column,
new HashMap<String, Object>() {
{
put("description", fields);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/HcatException.java Mon Jul 29 21:08:03 2013
@@ -18,6 +18,8 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
import java.util.HashMap;
/**
@@ -28,7 +30,7 @@ public class HcatException extends Simpl
public String statement;
public HcatException(String msg, final ExecBean bean, final String statement) {
- super(500, msg, new HashMap<String, Object>() {
+ super(HttpStatus.INTERNAL_SERVER_ERROR_500, msg, new HashMap<String, Object>() {
{
put("exec", bean);
put("statement", statement);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JsonBuilder.java Mon Jul 29 21:08:03 2013
@@ -25,17 +25,34 @@ import java.util.Map;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hcatalog.templeton.tool.TempletonUtils;
import org.codehaus.jackson.map.ObjectMapper;
+import org.eclipse.jetty.http.HttpStatus;
/**
* Helper class to build new json objects with new top level
* properties. Only add non-null entries.
*/
public class JsonBuilder {
- static final int OK = 200;
- static final int MISSING = 404;
- static final int SERVER_ERROR = 500;
+ private static final Map<Object, Integer> hiveError2HttpStatusCode = new HashMap<Object, Integer>();
+
+ /**
+ * It's expected that Hive (and thus HCat CLI) will return canonical error msgs/codes.
+ * Here they are mapped to appropriate HTTP Status Code.
+ */
+ static {
+ hiveError2HttpStatusCode.put(ErrorMsg.GENERIC_ERROR.getErrorCode(), HttpStatus.INTERNAL_SERVER_ERROR_500);
+ hiveError2HttpStatusCode.put(ErrorMsg.DATABASE_NOT_EXISTS.getErrorCode(), HttpStatus.NOT_FOUND_404);
+ hiveError2HttpStatusCode.put(ErrorMsg.INVALID_TABLE.getErrorCode(), HttpStatus.NOT_FOUND_404);
+ hiveError2HttpStatusCode.put(ErrorMsg.TABLE_NOT_PARTITIONED.getErrorCode(), HttpStatus.NOT_FOUND_404);
+ hiveError2HttpStatusCode.put(ErrorMsg.INVALID_PARTITION.getErrorCode(), HttpStatus.NOT_FOUND_404);
+
+ hiveError2HttpStatusCode.put(ErrorMsg.DUPLICATE_COLUMN_NAMES.getErrorCode(), HttpStatus.CONFLICT_409);
+ hiveError2HttpStatusCode.put(ErrorMsg.DATABSAE_ALREADY_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
+ hiveError2HttpStatusCode.put(ErrorMsg.PARTITION_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
+ hiveError2HttpStatusCode.put(ErrorMsg.TABLE_ALREADY_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
+ }
// The map we're building.
private Map map;
@@ -65,11 +82,11 @@ public class JsonBuilder {
/**
* Create a new map error object.
*/
- public static JsonBuilder createError(String msg, int code)
+ public static JsonBuilder createError(String msg, int errorCode)
throws IOException {
return new JsonBuilder(null)
.put("error", msg)
- .put("errorCode", code);
+ .put("errorCode", errorCode);
}
/**
@@ -115,16 +132,13 @@ public class JsonBuilder {
* Turn the map back to response object.
*/
public Response buildResponse() {
- int status = OK; // Server ok.
+ int status = HttpStatus.OK_200; // Server ok.
if (map.containsKey("error"))
- status = SERVER_ERROR; // Generic http server error.
+ status = HttpStatus.INTERNAL_SERVER_ERROR_500; // Generic http server error.
Object o = map.get("errorCode");
if (o != null) {
- try {
- status = Integer.parseInt(o.toString());
- } catch (Exception e) {
- if (o instanceof Number)
- status = ((Number) o).intValue();
+ if(hiveError2HttpStatusCode.containsKey(o)) {
+ status = hiveError2HttpStatusCode.get(o);
}
}
return buildResponse(status);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java Mon Jul 29 21:08:03 2013
@@ -41,8 +41,7 @@ public class ListDelegator extends Templ
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
TempletonJobTracker tracker = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
ArrayList<String> ids = new ArrayList<String>();
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Main.java Mon Jul 29 21:08:03 2013
@@ -50,6 +50,7 @@ public class Main {
private static final Log LOG = LogFactory.getLog(Main.class);
public static final int DEFAULT_PORT = 8080;
+ private Server server;
private static volatile AppConfig conf;
@@ -62,7 +63,7 @@ public class Main {
return conf;
}
- public Main(String[] args) {
+ Main(String[] args) {
init(args);
}
@@ -115,6 +116,17 @@ public class Main {
System.exit(1);
}
}
+ void stop() {
+ if(server != null) {
+ try {
+ server.stop();
+ }
+ catch(Exception ex) {
+ LOG.warn("Failed to stop jetty.Server", ex);
+ }
+ }
+ }
+
private void checkEnv() {
checkCurrentDirPermissions();
@@ -179,6 +191,7 @@ public class Main {
// Start the server
server.start();
+ this.server = server;
return server;
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/NotAuthorizedException.java Mon Jul 29 21:08:03 2013
@@ -18,11 +18,13 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
/**
* Simple "user not found" type exception.
*/
public class NotAuthorizedException extends SimpleWebException {
public NotAuthorizedException(String msg) {
- super(401, msg);
+ super(HttpStatus.UNAUTHORIZED_401, msg);
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/QueueException.java Mon Jul 29 21:08:03 2013
@@ -18,12 +18,14 @@
*/
package org.apache.hcatalog.templeton;
+import org.eclipse.jetty.http.HttpStatus;
+
/**
* Unable to queue the job
*/
public class QueueException extends SimpleWebException {
public QueueException(String msg) {
- super(500, msg);
+ super(HttpStatus.INTERNAL_SERVER_ERROR_500, msg);
}
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java Mon Jul 29 21:08:03 2013
@@ -44,8 +44,7 @@ public class StatusDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
throw new BadParam("Invalid jobid: " + id);
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Mon Jul 29 21:08:03 2013
@@ -18,11 +18,6 @@
*/
package org.apache.hcatalog.templeton;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-
/**
* The helper class for all the Templeton delegator classes. A
* delegator will call the underlying Templeton service such as hcat
@@ -34,10 +29,4 @@ public class TempletonDelegator {
public TempletonDelegator(AppConfig appConf) {
this.appConf = appConf;
}
-
- public static InetSocketAddress getAddress(Configuration conf) {
- String jobTrackerStr =
- conf.get("mapred.job.tracker", "localhost:8012");
- return NetUtils.createSocketAddr(jobTrackerStr);
- }
}
Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java (original)
+++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonControllerJob.java Mon Jul 29 21:08:03 2013
@@ -44,7 +44,6 @@ import org.apache.hadoop.io.NullWritable
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.Mapper;
@@ -68,9 +67,6 @@ import org.apache.hadoop.util.ToolRunner
* in hdfs files.
*/
public class TempletonControllerJob extends Configured implements Tool {
- static enum ControllerCounters {SIMPLE_COUNTER}
-
- ;
public static final String COPY_NAME = "templeton.copy";
public static final String STATUSDIR_NAME = "templeton.statusdir";
public static final String JAR_ARGS_NAME = "templeton.args";
@@ -157,14 +153,13 @@ public class TempletonControllerJob exte
conf.get(OVERRIDE_CLASSPATH));
String statusdir = conf.get(STATUSDIR_NAME);
- Counter cnt = context.getCounter(ControllerCounters.SIMPLE_COUNTER);
ExecutorService pool = Executors.newCachedThreadPool();
executeWatcher(pool, conf, context.getJobID(),
proc.getInputStream(), statusdir, STDOUT_FNAME);
executeWatcher(pool, conf, context.getJobID(),
proc.getErrorStream(), statusdir, STDERR_FNAME);
- KeepAlive keepAlive = startCounterKeepAlive(pool, cnt);
+ KeepAlive keepAlive = startCounterKeepAlive(pool, context);
proc.waitFor();
keepAlive.sendReport = false;
@@ -193,7 +188,7 @@ public class TempletonControllerJob exte
pool.execute(w);
}
- private KeepAlive startCounterKeepAlive(ExecutorService pool, Counter cnt)
+ private KeepAlive startCounterKeepAlive(ExecutorService pool, Context cnt)
throws IOException {
KeepAlive k = new KeepAlive(cnt);
pool.execute(k);
@@ -215,7 +210,7 @@ public class TempletonControllerJob exte
}
}
- public static class Watcher implements Runnable {
+ private static class Watcher implements Runnable {
private InputStream in;
private OutputStream out;
private JobID jobid;
@@ -279,11 +274,11 @@ public class TempletonControllerJob exte
}
}
- public static class KeepAlive implements Runnable {
- private Counter cnt;
- public boolean sendReport;
+ private static class KeepAlive implements Runnable {
+ private final Mapper.Context cnt;
+ private volatile boolean sendReport;
- public KeepAlive(Counter cnt) {
+ public KeepAlive(Mapper.Context cnt) {
this.cnt = cnt;
this.sendReport = true;
}
@@ -292,7 +287,7 @@ public class TempletonControllerJob exte
public void run() {
try {
while (sendReport) {
- cnt.increment(1);
+ cnt.progress();
Thread.sleep(KEEP_ALIVE_MSEC);
}
} catch (InterruptedException e) {
Modified: hive/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/ivy/libraries.properties?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/ivy/libraries.properties (original)
+++ hive/branches/tez/ivy/libraries.properties Mon Jul 29 21:08:03 2013
@@ -24,18 +24,17 @@ antlr.version=3.4
antlr-runtime.version=3.4
asm.version=3.1
avro.version=1.7.1
-datanucleus-connectionpool.version=2.0.3
-datanucleus-core.version=2.0.3
-datanucleus-enhancer.version=2.0.3
-datanucleus-rdbms.version=2.0.3
+datanucleus-api-jdo.version=3.2.1
+datanucleus-core.version=3.2.2
+datanucleus-rdbms.version=3.2.1
checkstyle.version=5.0
findbugs.version=1.3.9
+BoneCP.version=0.7.1.RELEASE
commons-cli.version=1.2
commons-codec.version=1.4
commons-collections.version=3.2.1
commons-compress.version=1.4.1
commons-configuration.version=1.6
-commons-dbcp.version=1.4
commons-httpclient.version=3.0.1
commons-io.version=2.4
commons-lang.version=2.4
@@ -47,7 +46,7 @@ guava.version=11.0.2
hbase.version=0.94.6.1
jackson.version=1.8.8
javaewah.version=0.3.2
-jdo-api.version=2.3-ec
+jdo-api.version=3.0.1
jdom.version=1.1
jetty.version=6.1.26
jline.version=0.9.94
Modified: hive/branches/tez/jdbc/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/build.xml?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/build.xml (original)
+++ hive/branches/tez/jdbc/build.xml Mon Jul 29 21:08:03 2013
@@ -33,7 +33,8 @@
<fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
<fileset dir="${hive.root}" includes="testlibs/*.jar"/>
<pathelement location="${build.dir.hive}/ql/test/classes"/>
- <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+ <fileset dir="${build.ivy.lib.dir}" includes="default/*.jar"/>
+ <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
<path refid="classpath" />
</distinctelementsclasspath>
</target>
Modified: hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java Mon Jul 29 21:08:03 2013
@@ -219,12 +219,20 @@ public abstract class HiveBaseResultSet
return null;
}
+ if (obj instanceof Date) {
+ return (Date) obj;
+ }
+
try {
- return Date.valueOf((String) obj);
+ if (obj instanceof String) {
+ return Date.valueOf((String)obj);
+ }
} catch (Exception e) {
throw new SQLException("Cannot convert column " + columnIndex
+ " to date: " + e.toString());
}
+
+ throw new SQLException("Illegal conversion");
}
public Date getDate(String columnName) throws SQLException {
Modified: hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HivePreparedStatement.java Mon Jul 29 21:08:03 2013
@@ -496,8 +496,7 @@ public class HivePreparedStatement imple
*/
public void setDate(int parameterIndex, Date x) throws SQLException {
- // TODO Auto-generated method stub
- throw new SQLException("Method not supported");
+ this.parameters.put(parameterIndex, x.toString());
}
/*
Modified: hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java Mon Jul 29 21:08:03 2013
@@ -110,6 +110,8 @@ public class HiveResultSetMetaData imple
return serdeConstants.INT_TYPE_NAME;
} else if ("bigint".equalsIgnoreCase(type)) {
return serdeConstants.BIGINT_TYPE_NAME;
+ } else if ("date".equalsIgnoreCase(type)) {
+ return serdeConstants.DATE_TYPE_NAME;
} else if ("timestamp".equalsIgnoreCase(type)) {
return serdeConstants.TIMESTAMP_TYPE_NAME;
} else if ("decimal".equalsIgnoreCase(type)) {
Modified: hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java Mon Jul 29 21:08:03 2013
@@ -74,6 +74,8 @@ public class JdbcColumn {
case Types.INTEGER:
case Types.BIGINT:
return columnPrecision(columnType) + 1; // allow +/-
+ case Types.DATE:
+ return 10;
case Types.TIMESTAMP:
return columnPrecision(columnType);
// see http://download.oracle.com/javase/6/docs/api/constant-values.html#java.lang.Float.MAX_EXPONENT
@@ -108,6 +110,8 @@ public class JdbcColumn {
return 7;
case Types.DOUBLE:
return 15;
+ case Types.DATE:
+ return 10;
case Types.TIMESTAMP:
return 29;
case Types.DECIMAL:
@@ -126,6 +130,7 @@ public class JdbcColumn {
case Types.SMALLINT:
case Types.INTEGER:
case Types.BIGINT:
+ case Types.DATE:
return 0;
case Types.FLOAT:
return 7;
Modified: hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hadoop/hive/jdbc/Utils.java Mon Jul 29 21:08:03 2013
@@ -46,6 +46,8 @@ public class Utils {
return Types.INTEGER;
} else if ("bigint".equalsIgnoreCase(type)) {
return Types.BIGINT;
+ } else if ("date".equalsIgnoreCase(type)) {
+ return Types.DATE;
} else if ("timestamp".equalsIgnoreCase(type)) {
return Types.TIMESTAMP;
} else if ("decimal".equalsIgnoreCase(type)) {
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java Mon Jul 29 21:08:03 2013
@@ -224,13 +224,20 @@ public abstract class HiveBaseResultSet
if (obj == null) {
return null;
}
-
+ if (obj instanceof Date) {
+ return (Date) obj;
+ }
try {
- return Date.valueOf((String) obj);
+ if (obj instanceof String) {
+ return Date.valueOf((String)obj);
+ }
} catch (Exception e) {
throw new SQLException("Cannot convert column " + columnIndex
+ " to date: " + e.toString(), e);
}
+ // If we fell through to here this is not a valid type conversion
+ throw new SQLException("Cannot convert column " + columnIndex
+ + " to date: Illegal conversion");
}
public Date getDate(String columnName) throws SQLException {
@@ -434,6 +441,15 @@ public abstract class HiveBaseResultSet
return null;
}
+ private Date getDateValue(TStringValue tStringValue) {
+ if (tStringValue.isSetValue()) {
+ wasNull = false;
+ return Date.valueOf(tStringValue.getValue());
+ }
+ wasNull = true;
+ return null;
+ }
+
private Timestamp getTimestampValue(TStringValue tStringValue) {
if (tStringValue.isSetValue()) {
wasNull = false;
@@ -495,10 +511,15 @@ public abstract class HiveBaseResultSet
return getStringValue(tColumnValue.getStringVal());
case BINARY_TYPE:
return getBinaryValue(tColumnValue.getStringVal());
+ case DATE_TYPE:
+ return getDateValue(tColumnValue.getStringVal());
case TIMESTAMP_TYPE:
return getTimestampValue(tColumnValue.getStringVal());
case DECIMAL_TYPE:
return getBigDecimalValue(tColumnValue.getStringVal());
+ case NULL_TYPE:
+ wasNull = true;
+ return null;
default:
throw new SQLException("Unrecognized column type:" + columnType);
}
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveCallableStatement.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveCallableStatement.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveCallableStatement.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveCallableStatement.java Mon Jul 29 21:08:03 2013
@@ -462,6 +462,16 @@ public class HiveCallableStatement imple
throw new SQLException("Method not supported");
}
+ public <T> T getObject(int parameterIndex, Class<T> type) throws SQLException {
+ // TODO JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
+ public <T> T getObject(String parameterName, Class<T> type) throws SQLException {
+ // TODO JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
/*
* (non-Javadoc)
*
@@ -2030,6 +2040,16 @@ public class HiveCallableStatement imple
throw new SQLException("Method not supported");
}
+ public void closeOnCompletion() throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
+ public boolean isCloseOnCompletion() throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
/*
* (non-Javadoc)
*
Modified: hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1508202&r1=1508201&r2=1508202&view=diff
==============================================================================
--- hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original)
+++ hive/branches/tez/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Mon Jul 29 21:08:03 2013
@@ -33,6 +33,7 @@ import java.sql.SQLXML;
import java.sql.Savepoint;
import java.sql.Statement;
import java.sql.Struct;
+import java.util.concurrent.Executor;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -160,27 +161,32 @@ public class HiveConnection implements j
}
}
- private void openSession(String uri) throws SQLException {
- TOpenSessionReq openReq = new TOpenSessionReq();
+ private void openSession(String uri) throws SQLException {
+ TOpenSessionReq openReq = new TOpenSessionReq();
- // set the session configuration
- // openReq.setConfiguration(null);
+ // set the session configuration
+ // openReq.setConfiguration(null);
- try {
- TOpenSessionResp openResp = client.OpenSession(openReq);
+ try {
+ TOpenSessionResp openResp = client.OpenSession(openReq);
- // validate connection
- Utils.verifySuccess(openResp.getStatus());
- if (!supportedProtocols.contains(openResp.getServerProtocolVersion())) {
- throw new TException("Unsupported Hive2 protocol");
- }
- sessHandle = openResp.getSessionHandle();
- } catch (TException e) {
- throw new SQLException("Could not establish connection to "
- + uri + ": " + e.getMessage(), " 08S01", e);
+ // validate connection
+ Utils.verifySuccess(openResp.getStatus());
+ if (!supportedProtocols.contains(openResp.getServerProtocolVersion())) {
+ throw new TException("Unsupported Hive2 protocol");
}
- isClosed = false;
+ sessHandle = openResp.getSessionHandle();
+ } catch (TException e) {
+ throw new SQLException("Could not establish connection to "
+ + uri + ": " + e.getMessage(), " 08S01", e);
}
+ isClosed = false;
+ }
+
+ public void abort(Executor executor) throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
/*
* (non-Javadoc)
@@ -395,6 +401,17 @@ public class HiveConnection implements j
return new HiveDatabaseMetaData(client, sessHandle);
}
+ public int getNetworkTimeout() throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
+
+ public String getSchema() throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
/*
* (non-Javadoc)
*
@@ -663,6 +680,11 @@ public class HiveConnection implements j
throw new SQLException("Method not supported");
}
+ public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
/*
* (non-Javadoc)
*
@@ -696,6 +718,11 @@ public class HiveConnection implements j
throw new SQLException("Method not supported");
}
+ public void setSchema(String schema) throws SQLException {
+ // JDK 1.7
+ throw new SQLException("Method not supported");
+ }
+
/*
* (non-Javadoc)
*