You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2013/01/01 04:01:40 UTC
svn commit: r1427292 - in /incubator/hcatalog/trunk: ./
src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/
src/test/e2e/hcatalog/tests/ src/test/e2e/hcatalog/tools/generate/
src/test/e2e/hcatalog/udfs/java/
Author: gates
Date: Tue Jan 1 04:01:40 2013
New Revision: 1427292
URL: http://svn.apache.org/viewvc?rev=1427292&view=rev
Log:
HCATALOG-583 e2e tests do not build
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl
incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue Jan 1 04:01:40 2013
@@ -168,6 +168,8 @@ Release 0.5.0 - Unreleased
OPTIMIZATIONS
BUG FIXES
+ HCAT-583 e2e tests do not build (gates)
+
HCAT-577 HCatContext causes persistance of undesired jobConf parameters (khorgath via gates)
HCAT-584 Changes in HCAT-538 break Pig stores into non-partitioned tables (gates)
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Tue Jan 1 04:01:40 2013
@@ -56,6 +56,7 @@ $cfg = {
, 'hcatlib' => "$ENV{'HCAT_HOME'}/share/hcatalog/lib"
, 'hcatconf' => "$ENV{'HCAT_HOME'}/etc/hcatalog"
, 'hcatbin' => "$ENV{'HCAT_HOME'}/bin/hcat"
+ , 'hcatcoredevlib' => "../../../../../core/build/lib/test"
#PIG
, 'pighome' => $ENV{'PIG_HOME'}
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Tue Jan 1 04:01:40 2013
@@ -254,8 +254,8 @@ sub runHadoop
# Get all of the additional jars we'll need.
my $additionalJars = Util::getHBaseLibs($testCmd, $log); #hbase before hive for precedence over bundled hbase
- $additionalJars .= Util::getHiveLibs($testCmd, $log);
$additionalJars .= Util::getHCatLibs($testCmd, $log);
+ $additionalJars .= Util::getHiveLibs($testCmd, $log);
$testCmd->{'libjars'} = $additionalJars;
$testCmd->{'libjars'} =~ s/:/,/g;
my $hadoopcmd = Util::replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log );
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Tue Jan 1 04:01:40 2013
@@ -412,10 +412,10 @@ sub getHiveLibsForPig($$)
/hive-.*\.jar$/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
- /libfb303.jar/ && do {
+ /libfb303-.*\.jar/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
- /libthrift.jar/ && do {
+ /libthrift-.*\.jar/ && do {
$cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
};
/datanucleus-.*\.jar$/ && do {
@@ -474,7 +474,7 @@ sub getHCatLibs($$)
opendir(LIB, $cfg->{'hcatshare'}) or die "Cannot open $cfg->{'hcatshare'}, $!\n";
my @jars = readdir(LIB);
foreach (@jars) {
- /hcatalog-[0-9].*\.jar$/ && do {
+ (/hcatalog-core-[0-9].*\.jar$/ || /hcatalog-pig-adapter-[0-9].*\.jar$/) && do {
$cp .= $cfg->{'hcatshare'} . '/' . $_ . ':';
};
}
@@ -487,6 +487,18 @@ sub getHCatLibs($$)
};
}
closedir(LIB);
+
+ # Get jars required non-hcat jars that are not distributed with Hadoop or Hive
+ opendir(LIB, $cfg->{'hcatcoredevlib'}) or die "Cannot open $cfg->{'hcatcoredevlib'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /guava.*\.jar$/ && do {
+ $cp .= $cfg->{'hcatcoredevlib'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+
+
return $cp;
}
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf Tue Jan 1 04:01:40 2013
@@ -137,7 +137,7 @@ $cfg = {
'num' => 4,
'sql' => q?
drop table if exists hive_read_4;
-create table hive_read_4 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
+create external table hive_read_4 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
alter table hive_read_4 add partition (b='1') location '/user/hcat/tests/data/studenttab10k';
alter table hive_read_4 set fileformat rcfile;
alter table hive_read_4 add partition (b='2') location '/user/hcat/tests/data/all100krc';
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Tue Jan 1 04:01:40 2013
@@ -161,7 +161,7 @@ store b into ':OUTPATH:';\,
# A table with one partition in text and one in rc
'num' => 4
,'hcat_prep'=>q?drop table if exists pig_read_4;
-create table pig_read_4 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
+create external table pig_read_4 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
alter table pig_read_4 add partition (b='1') location '/user/hcat/tests/data/studenttab10k';
alter table pig_read_4 set fileformat rcfile;
alter table pig_read_4 add partition (b='2') location '/user/hcat/tests/data/all100krc';?
@@ -177,7 +177,7 @@ store b into ':OUTPATH:';\,
'num' => 5
,'hcat_prep'=>q?create database if not exists pig_db_1;
drop table if exists pig_db_1.pig_read_5;
-create table pig_db_1.pig_read_5 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
+create external table pig_db_1.pig_read_5 (name string, age int, gpa double) partitioned by (b string) row format delimited fields terminated by '\t' stored as textfile;
use pig_db_1;
alter table pig_read_5 add partition (b='1') location '/user/hcat/tests/data/studenttab10k';?
,'pig' => q\a = load 'pig_db_1.pig_read_5' using org.apache.hcatalog.pig.HCatLoader();
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl Tue Jan 1 04:01:40 2013
@@ -330,7 +330,7 @@ location '$location'
sub findAllJars()
{
- my @files = <../../../../../build/ivy/lib/default/*.jar>;
+ my @files = <../../../../../core/build/lib/test/*.jar>;
my $classpath = "";
my $file = undef;
foreach $file (@files) {
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1427292&r1=1427291&r2=1427292&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml Tue Jan 1 04:01:40 2013
@@ -22,10 +22,15 @@
<property name="udf.jarfile" value="testudf.jar"/>
<property name="udfs.build.dir" value="${basedir}/build"/>
<property name="udfs.src.dir" value="${basedir}/org/"/>
+ <property name="hcatalog.base" value="../../../../../.."/>
+ <property name="hcatalog.core.build" value="${hcatalog.base}/core/build"/>
+ <property name="hcatalog.pig.build" value="${hcatalog.base}/hcatalog-pig-adapter/build"/>
<path id="udf-classpath">
- <fileset file="../../../../../../build/hcatalog/*.jar"/>
- <fileset file="../../../../../../build/ivy/lib/default/*.jar"/>
+ <fileset file="${hcatalog.core.build}/hcatalog-core-*.jar"/>
+ <fileset file="${hcatalog.core.build}/lib/compile/*.jar"/>
+ <fileset file="${hcatalog.pig.build}/hcatalog-pig-adapter-*.jar"/>
+ <fileset file="${hcatalog.pig.build}/lib/compile/*.jar"/>
</path>
<target name="init">