You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/04/12 07:51:04 UTC
svn commit: r1325149 - in /incubator/hcatalog/branches/branch-0.4: ./
src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/
src/test/e2e/hcatalog/drivers/
Author: gates
Date: Thu Apr 12 07:51:04 2012
New Revision: 1325149
URL: http://svn.apache.org/viewvc?rev=1325149&view=rev
Log:
HCATALOG-357 e2e harness: allow to run again pre-installed packages
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Thu Apr 12 07:51:04 2012
@@ -75,6 +75,8 @@ Release 0.4.0 - Unreleased
HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
IMPROVEMENTS
+ HCAT-357 e2e harness: allow to run again pre-installed packages (thw via gates)
+
HCAT-352 Need tests for complex types (gates)
HCAT-268 Remove remnants of storage drivers. (rohini via gates)
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml Thu Apr 12 07:51:04 2012
@@ -20,19 +20,7 @@
<property name="hcat.dir" value="${basedir}/../../../../"/>
<property name="hive.dir" value="${basedir}/../../../../hive/external/"/>
<property name="hcat.jar" value="${hcat.dir}/build/hcatalog/hcatalog-${hcatalog.version}.jar"/>
-
- <path id="hcat.deps">
- <fileset dir="${hive.dir}/lib/">
- <include name="*.jar" />
- </fileset>
- <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
- <include name="*.jar" />
- <exclude name="antlr*.jar" />
- </fileset>
- <pathelement location="${hcat.jar}" />
- </path>
- <property name="hcat-deps" refid="hcat.deps"/>
<!-- Separate property name for udfs' build.xml -->
<property name="hcat.jarfile" value="${hcat.jar}"/>
<property name="udf.dir" value="${basedir}/udfs"/>
@@ -53,6 +41,8 @@
<property name="harness.dir" value="${basedir}/../harness"/>
<property name="harness.tar" value="${harness.dir}/harness.tar"/>
+ <property name="harness.PH_LOCAL" value="."/>
+ <property name="harness.PH_OUT" value="."/>
<property name="test.location" value="${basedir}/testdist"/>
<property name="benchmark.location" value="${test.location}/benchmarks"/>
@@ -158,6 +148,77 @@
unless="harness.hadoop.home"/>
<fail message="Please set the property hive.metastore.uris to the hcat thrift server"
unless="hive.metastore.uris"/>
+ <fail message="Please set the property harness.pig.home to the path of your pig installation"
+ unless="harness.pig.home"/>
+
+ <dirset id="hcat.dist.lib.dir.fs" dir="${hcat.dir}/build" includes="hcatalog-*/share/hcatalog/lib"/>
+ <property name="hcat.dist.lib.dir" value="${hcat.dir}/build/${toString:hcat.dist.lib.dir.fs}"/>
+ <echo message="hcat libs: ${hcat.dist.lib.dir}"/>
+
+ <path id="hbase.jars.path">
+ <fileset dir="${hcat.dist.lib.dir}">
+ <include name="hbase-*.jar" />
+ </fileset>
+ </path>
+ <!-- override this for secure hbase -->
+ <property name="hbase.jars" refid="hbase.jars.path"/>
+
+ <path id="hcat.jars.path">
+ <fileset dir="${hcat.dir}/build/hcatalog">
+ <include name="hcatalog-*.jar" />
+ <exclude name="hcatalog-server-extensions-*.jar" />
+ </fileset>
+ </path>
+ <property name="hcat.jars" refid="hcat.jars.path"/>
+
+ <path id="hcat.client.deps">
+ <pathelement path="${hcat.jars}"/>
+ <pathelement path="${hbase.jars}"/>
+ <fileset dir="${hcat.dist.lib.dir}">
+ <include name="hbase-storage-handler-*.jar" />
+ <include name="zookeeper-*.jar" />
+ <include name="guava-*.jar" /><!-- for hbase storage handler -->
+ </fileset>
+ <!-- hbase-storage-handler only here with default ant target -->
+ <fileset dir="${hcat.dir}/storage-handlers/hbase/build/hbase-storage-handler-0.1.0/lib">
+ <include name="hbase-storage-handler-*.jar" />
+ <include name="zookeeper-*.jar" />
+ </fileset>
+ <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
+ <include name="hive-*.jar" />
+ <include name="libfb303-*.jar" />
+ <include name="guava-*.jar" />
+ <include name="antlr*.jar" />
+ </fileset>
+ </path>
+
+ <pathconvert pathsep="," property="hadoop.libjars" refid="hcat.client.deps"/>
+ <property name="hive.conf.dir" value="${hcat.install.dir}/etc/hcatalog"/>
+
+ <path id="pig.classpath">
+ <path refid="hcat.client.deps" />
+ <pathelement location="${hive.conf.dir}" />
+ </path>
+ <property name="pig.classpath" refid="pig.classpath"/>
+
+ <!-- copy of above w/o antlr -->
+ <path id="pig.additional.jars">
+ <pathelement path="${hcat.jars}"/>
+ <pathelement path="${hbase.jars}"/>
+ <fileset dir="${hcat.dist.lib.dir}">
+ <include name="hbase-storage-handler-*.jar" />
+ <include name="zookeeper-*.jar" />
+ <include name="guava-*.jar" /><!-- for hbase storage handler -->
+ </fileset>
+ <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
+ <include name="hive-*.jar" />
+ <include name="libfb303-*.jar" />
+ <!--include name="antlr*.jar" /-->
+ </fileset>
+ <pathelement location="${hive.conf.dir}" />
+ </path>
+ <property name="pig.additional.jars" refid="pig.additional.jars"/>
+
</target>
<!-- Prep the test area -->
@@ -176,20 +237,21 @@
</target>
<target name="test" depends="property-check, udfs, tar, init-test">
+
<!-- If they have not specified tests to run then null it out -->
<property name="tests.to.run" value=""/>
<echo />
<exec executable="./test_harness.pl" dir="${test.location}">
<env key="HARNESS_ROOT" value="."/>
- <env key="PH_LOCAL" value="."/>
- <env key="PH_OUT" value="."/>
+ <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
+ <env key="PH_OUT" value="${harness.PH_OUT}"/>
<env key="PH_ROOT" value="."/>
<env key="HCAT_ROOT" value="${hcat.dir}"/>
<env key="HCAT_INSTALL_DIR" value="${hcat.install.dir}"/>
<env key="HIVE_ROOT" value="${hcat.dir}/hive/external/"/>
- <env key="HCAT_EXTRA_JARS" value="${hcat-deps}/"/>
- <env key="PIG_CLASSPATH" value="${hcat-deps}/"/>
- <env key="HCAT_JAR" value="${hcat.jar}/"/>
+ <env key="PIG_CLASSPATH" value="${pig.classpath}/"/>
+ <env key="HADOOP_LIBJARS" value="${hadoop.libjars}"/>
+ <env key="HCAT_JARS" value="${hcat.jars}"/>
<env key="HADOOP_HOME" value="${harness.hadoop.home}/"/>
<env key="PH_OLDPIG" value="${harness.old.pig}"/>
<env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
@@ -198,8 +260,9 @@
<env key="HIVE_HOME" value="${basedir}/../../../../hive/external/build/dist"/>
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PIG_HOME" value="${harness.pig.home}"/>
- <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr -->
<env key="HBASE_CONF_DIR" value="${hbase.conf.dir}"/>
+ <env key="HIVE_CONF_DIR" value="${hive.conf.dir}"/>
+ <env key="PIG_ADDITIONAL_JARS" value="${pig.additional.jars}"/>
<arg line="${tests.to.run}"/>
<arg value="${test.location}/tests/pig.conf"/>
<arg value="${test.location}/tests/hive.conf"/>
@@ -221,14 +284,13 @@
<target name="deploy-base" depends="property-check, tar, init-test, init-deploy">
<exec executable="./test_harness.pl" dir="${test.location}">
<env key="HARNESS_ROOT" value="."/>
- <env key="PH_LOCAL" value="."/>
- <env key="PH_OUT" value="."/>
+ <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
+ <env key="PH_OUT" value="${harness.PH_OUT}"/>
<env key="PH_ROOT" value="."/>
<env key="HADOOP_HOME" value="${harness.hadoop.home}/"/>
<env key="HIVE_ROOT" value="${hcat.dir}/hive/external/"/>
<env key="HCAT_ROOT" value="${hcat.dir}"/>
<env key="HCAT_INSTALL_DIR" value="${hcat.install.dir}"/>
- <env key="HCAT_EXTRA_JARS" value="${hcat-deps}/"/>
<env key="PH_OLDPIG" value="${harness.old.pig}"/>
<env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf Thu Apr 12 07:51:04 2012
@@ -17,11 +17,6 @@
my $me = `whoami`;
chomp $me;
-#get the hive version from the hive build.properties
-my $hive_build_properties = "$ENV{HCAT_ROOT}/hive/external/build.properties";
-my $hive_version = `cat $hive_build_properties | grep '^version=' | cut -d= -f2`;
-chomp $hive_version;
-
# The contents of this file can be rewritten to fit your installation.
# Also, you can define the following environment variables and set things up as in the test setup
# PH_ROOT Root directory where test harness is installed
@@ -30,10 +25,12 @@ chomp $hive_version;
# PH_CLUSTER Root directory for cluster being used
# HCAT_ROOT Root directory for hcat version being used
+my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat";
+
$cfg = {
#HDFS
- 'inpathbase' => '/user/hcat/tests/data'
- , 'outpathbase' => '/user/hcat/out'
+ 'inpathbase' => "$hdfsBase/tests/data"
+ , 'outpathbase' => "$hdfsBase/out"
#LOCAL
, 'localinpathbase' => "$ENV{PH_LOCAL}/in"
@@ -48,10 +45,10 @@ $cfg = {
, 'jythonjar' => "$ENV{PH_JYTHON_JAR}"
#TESTDB
- , 'dbuser' => 'hcattest'
- , 'dbhost' => 'localhost'
- , 'dbpasswd' => 'hcattest'
- , 'dbdb' => 'hcattestdb'
+ , 'dbuser' => "$ENV{'PH_DBUSER'}" || 'hcattest'
+ , 'dbhost' => "$ENV{'PH_DBHOST'}" || 'localhost'
+ , 'dbpasswd' => "$ENV{'PH_DBPASSWD'}" || 'hcattest'
+ , 'dbdb' => "$ENV{'PH_DBDB'}" || 'hcattestdb'
#COMMON
, 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}"
@@ -62,7 +59,7 @@ $cfg = {
, 'hcat_data_dir' => '/user/hcat/tests/data'
, 'hivehome' => $ENV{'PH_HIVE_HOME'}
, 'hcathome' => $ENV{'HCAT_INSTALL_DIR'}
- , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
+ , 'hcatalog.jar' => $ENV{'HADOOP_LIBJARS'}
#PIG
, 'testconfigpath' => "$ENV{PH_CLUSTER}"
@@ -70,13 +67,11 @@ $cfg = {
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
, 'pigpath' => "$ENV{PIG_HOME}"
- , 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
- , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
+ , 'additionaljars' => "$ENV{PIG_ADDITIONAL_JARS}"
#HADOOP
- , 'hadoopHome' => "$ENV{HCAT_ROOT}/lib"
- , 'hadoop_classpath' => "$ENV{HCAT_EXTRA_JARS}"
+ , 'UNUSEDhadoopHome' => "$ENV{HCAT_ROOT}/lib"
, 'userhomePath' => "$ENV{HOME}"
, 'local.bin' => '/usr/bin'
, 'logDir' => "$ENV{PH_OUT}/log"
@@ -87,6 +82,7 @@ $cfg = {
, 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin"
, 'hbaseconfigpath' => "$ENV{HBASE_CONF_DIR}"
, 'hivehome' => "$ENV{HIVE_HOME}"
- , 'hive.additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar"
+ , 'hive.additionaljars' => "$ENV{HCAT_JARS}"
+ , 'hive.conf.dir' => "$ENV{HIVE_CONF_DIR}" || "$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
};
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf Thu Apr 12 07:51:04 2012
@@ -26,7 +26,7 @@ $cfg = {
# hadoop values
'hadoopdir' => $ENV{'PH_CLUSTER'},
- 'hcat_data_dir' => '/user/hcat/test/data',
+ 'hcat_data_dir' => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/test/data',
# db values
# 'dbuser' => 'pigtester',
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Thu Apr 12 07:51:04 2012
@@ -335,25 +335,18 @@ sub runHadoop
push(@cmd, split(/ +/,$hadoopcmd));
# Set HADOOP_CLASSPATH environment variable if provided
- if (defined($testCmd->{'hadoop_classpath'})) {
- #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log );
- # TODO This is a total mess. Half the jars we need are specified in hcatalog.jar, which is set in the default.conf file, and half are set in hadoop_classpath, which
- # has to be passed on the command line. And most of the jars are Hive jars, thus they don't fit in the list of either hcatalog.jar or hadoop_classpath. We need to
- # make sense of this.
- my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'};
- $cp =~ s/,/:/g;
- $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
- # Add in the hcat config file
- $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog";
- $ENV{'HADOOP_CLASSPATH'} = $cp;
- }
+ my $cp = $testCmd->{'hcatalog.jar'};
+ $cp =~ s/,/:/g;
+ # Add in the hcat config file
+ $cp .= ":" . $testCmd->{'hive.conf.dir'};
+ $ENV{'HADOOP_CLASSPATH'} = $cp;
if (defined($testCmd->{'hbaseconfigpath'})) {
$ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
}
if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) {
- $ENV{'HADOOP_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
+ $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
$ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
}
@@ -367,6 +360,7 @@ sub runHadoop
open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
print FH join (" ", @cmd) . "\n";
close(FH);
+
my @result=`chmod +x $script`;
# Run the command
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm Thu Apr 12 07:51:04 2012
@@ -161,7 +161,7 @@ sub runHiveCmdFromFile($$;$$$$)
# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false");
# }
- $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hcathome'}/etc/hcatalog/";
+ $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}";
if (defined($cfg->{'hive.additionaljars'})) {
$ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'};
@@ -560,13 +560,4 @@ sub getLocaleCmd
."export LC_IDENTIFICATION=\"$locale\"";
}
-sub findPigWithoutHadoopJar($$)
-{
- my ($cfg, $log) = @_;
-
- my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
- chomp $jar;
- return $jar;
-}
-
1;