You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/04/12 10:08:47 UTC
svn commit: r1325183 [2/3] - in /incubator/hcatalog/branches/branch-0.4: ./
bin/ hive/ ivy/ scripts/ src/docs/src/documentation/content/xdocs/
src/java/org/apache/hcatalog/cli/SemanticAnalysis/ src/test/e2e/hcatalog/
src/test/e2e/hcatalog/conf/ src/tes...
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Thu Apr 12 10:08:45 2012
@@ -52,52 +52,6 @@ sub new
return $self;
}
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
- my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
- # $self
- $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
- # $outfile
- $cmd =~ s/:OUTPATH:/$outfile/g;
-
- # $ENV
- $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
-
- # $testCmd
- $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
- $cmd =~ s/:OUTPATH:/$outfile/g;
- $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
- $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g;
- $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
- $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
- $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
- $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
- $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
- $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
-# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
-# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
-# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
- $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
- $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
- $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
-
- if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
- $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
- } else {
- $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
- }
-
- $cmd =~ s/:THRIFTSERVER:/$testCmd->{'thriftserver'}/g;
- $cmd =~ s/:HADOOP_CLASSPATH:/$testCmd->{'hadoop_classpath'}/g;
- $cmd =~ s/:HCAT_JAR:/$testCmd->{'hcatalog.jar'}/g;
-
- return $cmd;
-}
-
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
@@ -127,10 +81,7 @@ sub globalSetup
$ENV{'PATH'} = $globalHash->{'scriptPath'};
}
- my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
-
- print $log "Going to run " . join(" ", @cmd) . "\n";
- IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
+ Util::runHadoopCmd($globalHash, $log, "fs -mkdir $globalHash->{'outpath'}");
IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
die "Cannot create localpath directory " . $globalHash->{'localpath'} .
@@ -145,10 +96,7 @@ sub globalSetup
die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
- # Create the HDFS temporary directory
- @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
- print $log "Going to run " . join(" ", @cmd) . "\n";
- IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
+ Util::runHadoopCmd($globalHash, $log, "fs -mkdir tmp/$globalHash->{'runid'}");
}
sub globalCleanup
@@ -199,8 +147,8 @@ sub runTest
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
# Copy result file out of hadoop
- my @baseCmd = $self->getPigCmd($testCmd, $log);
- my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
+ my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
$outputs[$i] = $testOut;
$id++;
}
@@ -236,18 +184,16 @@ sub dumpPigTable
# Build the command
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
push(@cmd, $pigfile);
# Run the command
- print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
- IPC::Run::run(\@cmd, \undef, $log, $log) or
- die "Failed running $pigfile\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Failed running $pigfile\n";
$result{'rc'} = $? >> 8;
@@ -255,23 +201,17 @@ sub dumpPigTable
my $localoutfile;
my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
- $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
return $outfile;
}
sub postProcessSingleOutputFile
{
- my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_;
+ my ($self, $outfile, $localdir, $testCmd, $log) = @_;
my $subName = (caller(0))[3];
- my @baseCmd = @{$baseCmd};
- my @copyCmd = @baseCmd;
- push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir));
- print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
+ Util::runHadoopCmd($globalHash, $log, "fs -copyToLocal $outfile $localdir");
- IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
-
-
# Sort the result if necessary. Keep the original output in one large file.
# Use system not IPC run so that the '*' gets interpolated by the shell.
@@ -301,8 +241,6 @@ sub postProcessSingleOutputFile
}
sub runHadoop
-# Being modified from runPig
-# !!! Works, but need to add other arguments, like queue...???
{
my ($self, $testCmd, $log) = @_;
my $subName = (caller(0))[3];
@@ -313,7 +251,13 @@ sub runHadoop
my $hadoopfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".hadoop";
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- my $hadoopcmd = $self->replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log );
+ # Get all of the additional jars we'll need.
+ my $additionalJars = Util::getHBaseLibs($testCmd, $log); #hbase before hive for precedence over bundled hbase
+ $additionalJars .= Util::getHiveLibs($testCmd, $log);
+ $additionalJars .= Util::getHCatLibs($testCmd, $log);
+ $testCmd->{'libjars'} = $additionalJars;
+ $testCmd->{'libjars'} =~ s/:/,/g;
+ my $hadoopcmd = Util::replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log );
# adjust for the leading and trailing new line often seen in the conf file's command directives
$hadoopcmd =~ s/^\s*(.*?)\s*$/\1/s;
@@ -338,16 +282,12 @@ sub runHadoop
my $cp = $testCmd->{'hcatalog.jar'};
$cp =~ s/,/:/g;
# Add in the hcat config file
- $cp .= ":" . $testCmd->{'hive.conf.dir'};
+ $cp .= ":" . $testCmd->{'hiveconf'};
+ $cp .= ":" . $additionalJars;
$ENV{'HADOOP_CLASSPATH'} = $cp;
- if (defined($testCmd->{'hbaseconfigpath'})) {
- $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
- }
-
- if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) {
- $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
- $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
+ if (defined($testCmd->{'hbaseconf'})) {
+ $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconf'}";
}
# Add su user if provided
@@ -373,10 +313,10 @@ sub runHadoop
die "Failed running $script\n";
my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
if ($self->countStores($testCmd)==1) {
@outputs = ();
- $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
$result{'outputs'} = \@outputs;
}
@@ -535,7 +475,7 @@ sub runPig
my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+ my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
print FH $pigcmd . "\n";
@@ -543,7 +483,8 @@ sub runPig
# Build the command
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ #my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -580,7 +521,7 @@ sub runPig
# single query
if ($stores == 1) {
if ($copyResults) {
- $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
$result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
} else {
$result{'output'} = "NO_COPY";
@@ -596,7 +537,7 @@ sub runPig
# Copy result file out of hadoop
my $testOut;
if ($copyResults) {
- $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+ $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, $testCmd, $log);
} else {
$testOut = "NO_COPY";
}
@@ -615,47 +556,6 @@ sub runPig
return \%result;
}
-sub getPigCmd($$$)
-{
- my ($self, $testCmd, $log) = @_;
-
- my @pigCmd;
-
- # set the PIG_CLASSPATH environment variable
- my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
- $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
- $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
- # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
- $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'});
- $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne ""));
-
- # Set it in our current environment. It will get inherited by the IPC::Run
- # command.
- $ENV{'PIG_CLASSPATH'} = $pcp;
-
- @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
-
- if (defined($testCmd->{'additionaljars'})) {
- push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
- }
-
- if ($testCmd->{'exectype'} eq "local") {
- push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
- push(@pigCmd, ("-x", "local"));
- }
-
- my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}";
- if (defined($testCmd->{'java_params'})) {
- $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
- }
-
- $ENV{'PIG_OPTS'} = $opts;
-
- print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
- print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
- return @pigCmd;
-}
-
sub compareSingleOutput
{
my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHive.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHive.pm?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHive.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHive.pm Thu Apr 12 10:08:45 2012
@@ -53,35 +53,11 @@ sub new
return $self;
}
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
- my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
- # $self
- $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
- # $outfile
- $cmd =~ s/:OUTPATH:/$outfile/g;
-
- # $ENV
- $cmd =~ s/:HARNESS:/$ENV{HARNESS_ROOT}/g;
-
- # $testCmd
- $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
-
- return $cmd;
-}
-
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
my $subName = (caller(0))[3];
- # Set up values for the metastore
- Util::setupHiveProperties($globalHash, $log);
-
# Setup the output path
my $me = `whoami`;
chomp $me;
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Thu Apr 12 10:08:45 2012
@@ -52,48 +52,6 @@ sub new
return $self;
}
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
- my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
- # $self
- $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
- # $outfile
- $cmd =~ s/:OUTPATH:/$outfile/g;
-
- # $ENV
- $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
-
- # $testCmd
- $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
- $cmd =~ s/:OUTPATH:/$outfile/g;
- $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
- $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g;
- $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
- $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
- $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
- $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
- $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
- $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
-# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
-# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
-# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
- $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
- $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
- $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
-
- if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
- $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
- } else {
- $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
- }
-
- return $cmd;
-}
-
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
@@ -123,7 +81,7 @@ sub globalSetup
$ENV{'PATH'} = $globalHash->{'scriptPath'};
}
- my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
+ my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
print $log "Going to run " . join(" ", @cmd) . "\n";
@@ -143,7 +101,7 @@ sub globalSetup
" " . "$ERRNO\n";
# Create the HDFS temporary directory
- @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
+ @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
print $log "Going to run " . join(" ", @cmd) . "\n";
IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
@@ -157,7 +115,7 @@ sub globalCleanup
" " . "$ERRNO\n";
# Cleanup the HDFS temporary directory
- my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
+ my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
print $log "Going to run " . join(" ", @cmd) . "\n";
IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
@@ -221,7 +179,7 @@ sub runTest
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
# Copy result file out of hadoop
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
$outputs[$i] = $testOut;
$id++;
@@ -264,22 +222,16 @@ sub runPigCmdLine
}
# Write the pig script to a file.
- my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+ my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
print FH $pigcmd . "\n";
close(FH);
# Build the command
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
- # Add option -l giving location for secondary logs
- ##!!! Should that even be here?
- my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
- push(@cmd, "-logfile");
- push(@cmd, $locallog);
-
# Add pig parameters if they're provided
if (defined($testCmd->{'pig_params'})) {
# Processing :PARAMPATH: in parameters
@@ -333,7 +285,7 @@ sub runScript
}
# Write the script to a file
- my $cmd = $self->replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
+ my $cmd = Util::replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
print FH $cmd . "\n";
@@ -363,88 +315,6 @@ sub runScript
}
-sub getPigCmd($$$)
-{
- my ($self, $testCmd, $log) = @_;
-
- my @pigCmd;
-
- # set the PIG_CLASSPATH environment variable
- my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
- $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
- $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
- # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
- $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'});
- $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne ""));
-
- # Set it in our current environment. It will get inherited by the IPC::Run
- # command.
- $ENV{'PIG_CLASSPATH'} = $pcp;
-
- @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
-
- if (defined($testCmd->{'additionaljars'})) {
- push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
- }
-
- if ($testCmd->{'exectype'} eq "local") {
- push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
- push(@pigCmd, ("-x", "local"));
- }
-
- my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}";
- if (defined($testCmd->{'java_params'})) {
- $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
- }
-
- $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts;
-
- print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
- print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
- return @pigCmd;
-}
-
-sub dumpPigTable
-{
- my ($self, $testCmd, $table, $log, $id) = @_;
- my $subName = (caller(0))[3];
-
- my %result;
-
- # Write the pig script to a file.
- my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.pig";
- my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . "dump.out";
-
- open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
- print FH "a = load '$table' using org.apache.hcatalog.pig.HCatLoader(); store a into '$outfile';\n";
- close(FH);
-
-
- # Build the command
- my @baseCmd = $self->getPigCmd($testCmd, $log);
- my @cmd = @baseCmd;
-
- push(@cmd, $pigfile);
-
-
- # Run the command
- print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
- print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
-
- IPC::Run::run(\@cmd, \undef, $log, $log) or
- die "Failed running $pigfile\n";
- $result{'rc'} = $? >> 8;
-
-
- # Get results from the command locally
- my $localoutfile;
- my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
- my $stores = $self->countStores($testCmd);
-
- $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
- return $outfile;
-}
-
sub runPig
{
my ($self, $testCmd, $log, $copyResults, $noFailOnFail) = @_;
@@ -456,7 +326,7 @@ sub runPig
my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+ my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
print FH $pigcmd . "\n";
@@ -464,7 +334,7 @@ sub runPig
# Build the command
- my @baseCmd = $self->getPigCmd($testCmd, $log);
+ my @baseCmd = Util::getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -485,8 +355,9 @@ sub runPig
# Run the command
- print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
+ print $log "With PIG_CLASSPATH set to $ENV{'PIG_CLASSPATH'}\n";
+ print $log "and HADOOP_HOME set to $ENV{'HADOOP_HOME'}\n";
my $runrc = IPC::Run::run(\@cmd, \undef, $log, $log);
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm Thu Apr 12 10:08:45 2012
@@ -27,12 +27,14 @@
package Util;
use IPC::Run qw(run);
+use strict;
+use English;
sub prepareHCat
{
my ($self, $testCmd, $log) = @_;
my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- my $hcatCmd = $self->replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log);
+ my $hcatCmd = replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log);
my @hivefiles = ();
my @outfiles = ();
@@ -50,46 +52,6 @@ sub prepareHCat
Util::runHCatCmdFromFile($testCmd, $log, $hivefiles[0]);
}
-##############################################################################
-# Sub: setupHiveProperties
-#
-# Assure that necessary values are set in config in order to set Hive
-# Java properties.
-#
-# Returns:
-# Nothing
-sub setupHiveProperties($$)
-{
- my ($cfg, $log) = @_;
-
- # Set up values for the metastore
- if (defined($cfg->{'metastore_thrift'}) && $cfg->{'metastore_thrift'} == 1) {
- if (! defined $cfg->{'metastore_host'} || $cfg->{'metastore_host'} eq "") {
- print $log "When using thrift, you must set the key " .
- " 'metastore_host' to the machine your metastore is on\n";
- die "metastore_host is not set in existing.conf\n";
- }
-
- $cfg->{'metastore_connection'} =
- "jdbc:$cfg->{'metastore_db'}://$cfg->{'metastore_host'}/hivemetastoredb?createDatabaseIfNotExist=true";
-
- if (! defined $cfg->{'metastore_passwd'} || $cfg->{'metastore_passwd'} eq "") {
- $cfg->{'metastore_passwd'} = 'hive';
- }
-
- if (! defined $cfg->{'metastore_port'} || $cfg->{'metastore_port'} eq "") {
- $cfg->{'metastore_port'} = '9933';
- }
-
- $cfg->{'metastore_uri'} =
- "thrift://$cfg->{'metastore_host'}:$cfg->{'metastore_port'}";
- } else {
- $cfg->{'metastore_connection'} =
- "jdbc:derby:;databaseName=metastore_db;create=true";
- $cfg->{'metastore_driver'} = "org.apache.derby.jdbc.EmbeddedDriver";
- }
-}
-
sub getHadoopCmd
{
my ( $properties ) = @_;
@@ -113,7 +75,8 @@ sub getHadoopCmd
}
push (@baseCmd, $cmd);
- push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+ push (@baseCmd, '--config', $properties->{'hadoopconfdir'})
+ if defined($properties->{'hadoopconfdir'});
return @baseCmd;
}
@@ -148,31 +111,11 @@ sub runHiveCmdFromFile($$;$$$$)
$outfile = $log if (!defined($outfile));
$errfile = $log if (!defined($errfile));
- my @cmd = ("$cfg->{'hivehome'}/bin/hive");
-
- # Add all of the modified properties we want to set
-# push(@cmd, "--hiveconf", "hive.metastore.uris=$cfg->{'thriftserver'}");
-# push(@cmd, "--hiveconf", "hive.metastore.local=false");
-
-# if( defined($cfg->{'metastore.principal'}) && ($cfg->{'metastore.principal'} =~ m/\S+/)
-# && ($cfg->{'metastore.principal'} ne '${metastore.principal}')){
-# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=true", "--hiveconf", "hive.metastore.kerberos.principal=$cfg->{'metastore.principal'}");
-# } else {
-# push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false");
-# }
-
- $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}";
-
- if (defined($cfg->{'hive.additionaljars'})) {
- $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'};
- }
-
-# if (defined($cfg->{'hiveconf'})) {
-# foreach my $hc (@{$cfg->{'hiveconf'}}) {
-# push(@cmd, "--hiveconf", $hc);
-# }
-# }
-
+ my @cmd = ($cfg->{'hivebin'});
+
+ $ENV{'HIVE_CONF_DIR'} = $cfg->{'hiveconf'};
+ $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hcatshare'};
+
if (defined($cfg->{'hivecmdargs'})) {
push(@cmd, @{$cfg->{'hivecmdargs'}});
}
@@ -230,26 +173,28 @@ sub runHCatCmdFromFile($$;$$$$)
my ($cfg, $log, $sql, $outfile, $errfile, $noFailOnFail) = @_;
if (!defined($ENV{'HADOOP_HOME'})) {
- die "Cannot run hive when HADOOP_HOME environment variable is not set.";
+ die "Cannot run hcat when HADOOP_HOME environment variable is not set.";
}
$outfile = $log if (!defined($outfile));
$errfile = $log if (!defined($errfile));
# unset HADOOP_CLASSPATH
- $ENV{'HADOOP_CLASSPATH'} = "";
- $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'};
+# $ENV{'HADOOP_CLASSPATH'} = "";
+ $ENV{'HADOOP_CLASSPATH'} = $cfg->{'hbaseconf'};
+ $ENV{'HCAT_CLASSPATH'} = Util::getHBaseLibs($cfg, $log);
my @cmd;
if (defined($sql)) {
- @cmd = ("$cfg->{'hcathome'}/bin/hcat", "-f", $sql);
+ @cmd = ("$cfg->{'hcatbin'}", "-f", $sql);
} else {
- @cmd = ("$cfg->{'hcathome'}/bin/hcat");
+ @cmd = ("$cfg->{'hcatbin'}");
}
my $envStr;
for my $k (keys(%ENV)) {
- $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/);
+ $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/ ||
+ $k =~ /HCAT/);
}
$envStr .= " ";
print $log "Going to run hcat command [" . join(" ", @cmd) .
@@ -327,198 +272,33 @@ sub show_call_stack {
sub getPigCmd
{
- my $subName = (caller(0))[3];
- my $jarkey = shift;
- my ( $properties ) = @_;
- my $isPigSqlEnabled= 0;
- my @baseCmd;
- die "$0.$subName: null properties" if (! $properties );
-
-show_call_stack();
- #UGLY HACK for pig sql support
- if ( $jarkey =~ /testsql/ ) {
-
- $isPigSqlEnabled= 1;
- $jarkey = "testjar";
-
- }
-
- my $cmd;
- if ( $properties->{'use-pig.pl'} ) {
- # The directive gives that
- # 1) the 'pig' command will be called, as opposed to java
- # 2) the conf file has full control over what options and parameters are
- # passed to pig.
- # I.e. no parameters should be passed automatically by the script here.
- #
- # This allows for testing of the pig script as installed, and for testin of
- # the pig script's options, including error testing.
-
-print 'use-pig.pl?????';
-
- $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig";
- if ( ! -x "$cmd" ) {
- print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n";
- $cmd = `which pig`;
- chomp $cmd;
- print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
- }
- die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd );
- push (@baseCmd, $cmd );
-
- if(defined($properties->{'additionaljars'})) {
- push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
- }
- $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'};
-
- if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically
- # !!!
- return @baseCmd;
- }
-
- } else {
- $cmd="java";
+ my ( $cfg, $log ) = @_;
-print 'not use-pig.pl?????';
- # Set JAVA options
+ my @cmd = ("$cfg->{'pigbin'}");
- # User can provide only one of
- # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>)
- # "-c <cluster>" is allowed only in non local mode
- if(defined($properties->{'cluster.name'})) {
- # use provided cluster
- @baseCmd = ($cmd, '-c', $properties->{'cluster.name'});
- } else {
- die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) );
- die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e $properties->{$jarkey} );
-
- # use user provided jar
- my $classpath;
-
- if (defined $properties->{'jythonjar'}) {
- $classpath = "$classpath:" . $properties->{'jythonjar'};
- }
- if( $properties->{'exectype'} eq "local") {
- # in local mode, we should not use
- # any hadoop-site.xml
- $classpath= "$classpath:" . $properties->{$jarkey};
- $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
- @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m');
-
- } else {
-
- # non local mode, we also need to specify
- # location of hadoop-site.xml
- die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n"
- if ( !defined( $properties->{'testconfigpath'} ) );
- die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n"
- if (! -e $properties->{'testconfigpath'} );
-
- $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'};
- $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
- $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) );
- @baseCmd = ($cmd, '-cp', $classpath );
- }
- }
-
- # sets the queue, for exampel "grideng"
- if(defined($properties->{'queue'})) {
- push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'});
- }
-
- if(defined($properties->{'additionaljars'})) {
- push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
- }
-
- if( ( $isPigSqlEnabled == 1 ) ){
-
- if(defined($properties->{'metadata.uri'})) {
- push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'});
- }
-
- if(defined($properties->{'metadata.impl'})) {
- push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'});
- }else{
- push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables');
- }
- }
-
- # Add howl support
- if(defined($properties->{'howl.metastore.uri'})) {
- push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'});
- }
+ # sets the queue, for exampel "grideng"
+ if(defined($cfg->{'queue'})) {
+ push( @cmd,'-Dmapred.job.queue.name='.$cfg->{'queue'});
+ }
- # Set local mode property
- # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
- # Removed above 'if...' for Pig 8.
- my $java=`which java`;
- my $version=`file $java`;
- if ( $version =~ '32-bit' ){
- push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32');
- } else {
- push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64');
- }
- # }
-
-
- # Add user provided java options if they exist
- if (defined($properties->{'java_params'})) {
- push(@baseCmd, @{$properties->{'java_params'}});
- }
+ my $cp = Util::getHCatLibs($cfg, $log) . Util::getHiveLibsForPig($cfg, $log) .
+ Util::getHBaseLibs($cfg, $log);
+ push(@cmd, ('-Dpig.additional.jars='. $cp));
+ $cp .= ':' . $cfg->{'hiveconf'};
+ $cp .= ':' . $cfg->{'hbaseconf'};
+ $ENV{'PIG_CLASSPATH'} = $cp;
- if(defined($properties->{'hod'})) {
- push( @baseCmd, '-Dhod.server=');
- }
-
- # sets the permissions on the jobtracker for the logs
- push( @baseCmd,'-Dmapreduce.job.acl-view-job=*');
+ # sets the permissions on the jobtracker for the logs
+ push( @cmd,'-Dmapreduce.job.acl-view-job=*');
- # Add Main
- push(@baseCmd, 'org.apache.pig.Main');
-
# Set local mode PIG option
- if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
- push(@baseCmd, '-x');
- push(@baseCmd, 'local');
- }
-
- # Set Pig SQL options
- if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) {
-
- if ( defined($properties->{'testoutpath'}) ) {
- push( @baseCmd, '-u' );
- push( @baseCmd, $properties->{'testoutpath'} );
- }
-
- push( @baseCmd, '-s' );
- push( @baseCmd, '-f' );
- }
-
- } # end else of if use-pig.pl
-
-
- # Add -latest or -useversion
- if ( $cmd =~ 'pig$' ) {
- # Add -latest, or -useversion if 'current' is not target build
- if ( defined($properties->{'pigTestBuildName'})) {
- if ($properties->{'pigTestBuildName'} eq 'latest') {
- push(@baseCmd, '-latest');
- } elsif ($properties->{'pigTestBuildName'} ne 'current') {
- push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}");
- }
- }
- } elsif ( $cmd =~ 'java' ) {
-
- # is this ever used: ???
- # Add latest if it's there
- if (defined($properties->{'latest'})) {
- push(@baseCmd, '-latest');
- }
+ if ( defined($cfg->{'exectype'}) && $cfg->{'exectype'}=~ "local" ) {
+ push(@cmd, ('-x', 'local'));
}
- return @baseCmd;
+ return @cmd;
}
@@ -527,18 +307,18 @@ sub setLocale
my $locale= shift;
# $locale = "en_US.UTF-8" if ( !$locale );
$locale = "ja_JP.utf8" if ( !$locale );
- $ENV[LC_CTYPE]="$locale";
- $ENV[LC_NUMERIC]="$locale";
- $ENV[LC_TIME]="$locale";
- $ENV[LC_COLLATE]="$locale";
- $ENV[LC_MONETARY]="$locale";
- $ENV[LC_MESSAGES]="$locale";
- $ENV[LC_PAPER]="$locale";
- $ENV[LC_NAME]="$locale";
- $ENV[LC_ADDRESS]="$locale";
- $ENV[LC_TELEPHONE]="$locale";
- $ENV[LC_MEASUREMENT]="$locale";
- $ENV[LC_IDENTIFICATION]="$locale";
+ $ENV['LC_CTYPE']="$locale";
+ $ENV['LC_NUMERIC']="$locale";
+ $ENV['LC_TIME']="$locale";
+ $ENV['LC_COLLATE']="$locale";
+ $ENV['LC_MONETARY']="$locale";
+ $ENV['LC_MESSAGES']="$locale";
+ $ENV['LC_PAPER']="$locale";
+ $ENV['LC_NAME']="$locale";
+ $ENV['LC_ADDRESS']="$locale";
+ $ENV['LC_TELEPHONE']="$locale";
+ $ENV['LC_MEASUREMENT']="$locale";
+ $ENV['LC_IDENTIFICATION']="$locale";
}
sub getLocaleCmd
@@ -560,4 +340,154 @@ sub getLocaleCmd
."export LC_IDENTIFICATION=\"$locale\"";
}
+sub replaceParameters
+{
+
+ my ($cmd, $outfile, $testCmd, $log) = @_;
+
+ # $self
+# $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+ # $outfile
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+
+ # $ENV
+ $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
+
+ # $testCmd
+ $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+ $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g;
+ $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
+ $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
+ $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
+ $cmd =~ s/:HCAT_JAR:/$testCmd->{'libjars'}/g;
+
+ if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
+ $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
+ } else {
+ $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
+ }
+
+ return $cmd;
+}
+
+sub getHiveLibs($$)
+{
+ my ($cfg, $log) = @_;
+
+ my $cp;
+ opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+ return $cp;
+}
+
+# Pig needs a limited set of the Hive libs, since they include some of the same jars
+# and we get version mismatches if it picks up all the libraries.
+sub getHiveLibsForPig($$)
+{
+ my ($cfg, $log) = @_;
+
+ my $cp;
+ opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /hive-.*\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /libfb303.jar/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /libthrift.jar/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /datanucleus-.*\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /jdo2-api-.*\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /commons-dbcp-.*\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+ /commons-pool-.*\.jar$/ && do {
+ $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+ };
+# /hbase-.*\.jar$/ && do {
+# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+# };
+# /zookeeper-.*\.jar$/ && do {
+# $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+# };
+ }
+ closedir(LIB);
+ return $cp;
+}
+
+sub getHBaseLibs($$)
+{
+ my ($cfg, $log) = @_;
+
+ my $cp;
+ opendir(LIB, $cfg->{'hbaselibdir'}) or die "Cannot open $cfg->{'hbaselibdir'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /hbase-.*\.jar$/ && do {
+ $cp .= $cfg->{'hbaselibdir'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+ opendir(LIB, $cfg->{'zklibdir'}) or die "Cannot open $cfg->{'zklibdir'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /zookeeper.*\.jar$/ && do {
+ $cp .= $cfg->{'zklibdir'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+ return $cp;
+}
+
+
+sub getHCatLibs($$)
+{
+ my ($cfg, $log) = @_;
+
+ my $cp;
+ opendir(LIB, $cfg->{'hcatshare'}) or die "Cannot open $cfg->{'hcatshare'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /hcatalog-[0-9].*\.jar$/ && do {
+ $cp .= $cfg->{'hcatshare'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+ opendir(LIB, $cfg->{'hcatlib'}) or die "Cannot open $cfg->{'hcatlib'}, $!\n";
+ my @jars = readdir(LIB);
+ foreach (@jars) {
+ /hbase-storage-handler.*\.jar$/ && do {
+ $cp .= $cfg->{'hcatlib'} . '/' . $_ . ':';
+ };
+ }
+ closedir(LIB);
+ return $cp;
+}
+
+
1;
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl Thu Apr 12 10:08:45 2012
@@ -361,8 +361,8 @@ sub findHiveJars()
die 'Please set $HIVE_HOME\n';
}
- my $execJar = `ls ../../../../../hive/external/build/ql/hive-exec-*.jar`;
- my $cliJar = `ls ../../../../../hive/external/build/cli/hive-cli-*.jar`;
+ my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`;
+ my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`;
chomp $execJar;
chomp $cliJar;
return ($execJar, $cliJar);
Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/build.xml Thu Apr 12 10:08:45 2012
@@ -22,8 +22,6 @@
<path id="udf-classpath">
<fileset file="../../../../../../build/hcatalog/*.jar" />
<fileset file="../../../../../../build/ivy/lib/hcatalog/*.jar" />
- <fileset file="../../../../../../hive/external/build/dist/lib/*.jar" />
- <fileset file="../../../../../../hive/external/build/hadoopcore/hadoop-0.20.1/*.jar" />
</path>
<target name="init">
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java Thu Apr 12 10:08:45 2012
@@ -18,8 +18,12 @@
package org.apache.hcatalog;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -28,11 +32,14 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hcatalog.data.Pair;
+import org.apache.pig.PigServer;
/**
* Utility methods for tests
*/
public class HcatTestUtils {
+ private static final Log LOG = LogFactory.getLog(HcatTestUtils.class);
public static FsPermission perm007 = FsPermission.createImmutable((short) 0007); // -------rwx
public static FsPermission perm070 = FsPermission.createImmutable((short) 0070); // ----rwx---
@@ -74,5 +81,21 @@ public class HcatTestUtils {
hive.dropTable("default", tablename, true, true);
}
}
-
+
+ public static void createTestDataFile(String filename, String[] lines) throws IOException {
+ FileWriter writer = null;
+ try {
+ File file = new File(filename);
+ file.deleteOnExit();
+ writer = new FileWriter(file);
+ for (String line : lines) {
+ writer.write(line + "\n");
+ }
+ } finally {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+
+ }
}
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatHiveCompatibility.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatHiveCompatibility.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatHiveCompatibility.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatHiveCompatibility.java Thu Apr 12 10:08:45 2012
@@ -18,13 +18,15 @@
package org.apache.hcatalog.mapreduce;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
-import java.util.Properties;
import junit.framework.TestCase;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -32,28 +34,29 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hcatalog.MiniCluster;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.pig.HCatLoader;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.util.UDFContext;
-
public class TestHCatHiveCompatibility extends TestCase {
+ private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
+ "/build/test/data/" + TestHCatHiveCompatibility.class.getCanonicalName();
+ private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
+ private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
- MiniCluster cluster = MiniCluster.buildCluster();
private Driver driver;
- Properties props;
-
private HiveMetaStoreClient client;
- String fileName = "/tmp/input.data";
- String fullFileName;
-
@Override
protected void setUp() throws Exception {
+ File f = new File(TEST_WAREHOUSE_DIR);
+ if (f.exists()) {
+ FileUtil.fullyDelete(f);
+ }
+
+ new File(TEST_WAREHOUSE_DIR).mkdirs();
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
@@ -62,22 +65,15 @@ public class TestHCatHiveCompatibility e
driver = new Driver(hiveConf);
client = new HiveMetaStoreClient(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
- props = new Properties();
- props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
- fullFileName = cluster.getProperties().getProperty("fs.default.name") + fileName;
- MiniCluster.deleteFile(cluster, fileName);
int LOOP_SIZE = 11;
- String[] input = new String[LOOP_SIZE];
+ File file = new File(INPUT_FILE_NAME);
+ file.deleteOnExit();
+ FileWriter writer = new FileWriter(file);
for(int i = 0; i < LOOP_SIZE; i++) {
- input[i] = i + "\t1";
+ writer.write(i + "\t1\n");
}
- MiniCluster.createInputFile(cluster, fileName, input);
- }
-
- @Override
- protected void tearDown() throws Exception {
- MiniCluster.deleteFile(cluster, fileName);
+ writer.close();
}
public void testUnpartedReadWrite() throws Exception{
@@ -93,9 +89,8 @@ public class TestHCatHiveCompatibility e
Table table = client.getTable("default", "junit_unparted_noisd");
assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS));
- PigServer server = new PigServer(ExecType.LOCAL, props);
- UDFContext.getUDFContext().setClientSystemProps();
- server.registerQuery("A = load '"+fullFileName+"' as (a:int);");
+ PigServer server = new PigServer(ExecType.LOCAL);
+ server.registerQuery("A = load '"+INPUT_FILE_NAME+"' as (a:int);");
server.registerQuery("store A into 'default.junit_unparted_noisd' using org.apache.hcatalog.pig.HCatStorer();");
server.registerQuery("B = load 'default.junit_unparted_noisd' using "+HCatLoader.class.getName()+"();");
Iterator<Tuple> itr= server.openIterator("B");
@@ -133,9 +128,8 @@ public class TestHCatHiveCompatibility e
assertTrue(table.getSd().getInputFormat().equals(HCatConstants.HIVE_RCFILE_IF_CLASS));
- PigServer server = new PigServer(ExecType.LOCAL, props);
- UDFContext.getUDFContext().setClientSystemProps();
- server.registerQuery("A = load '"+fullFileName+"' as (a:int);");
+ PigServer server = new PigServer(ExecType.LOCAL);
+ server.registerQuery("A = load '"+INPUT_FILE_NAME+"' as (a:int);");
server.registerQuery("store A into 'default.junit_parted_noisd' using org.apache.hcatalog.pig.HCatStorer('b=42');");
server.registerQuery("B = load 'default.junit_parted_noisd' using "+HCatLoader.class.getName()+"();");
Iterator<Tuple> itr= server.openIterator("B");
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestPassProperties.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestPassProperties.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestPassProperties.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestPassProperties.java Thu Apr 12 10:08:45 2012
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hcatalog.HcatTestUtils;
-import org.apache.hcatalog.MiniCluster;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.common.HCatUtil;
@@ -57,11 +56,10 @@ public class TestPassProperties {
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
- private static MiniCluster cluster = MiniCluster.buildCluster();
private static Driver driver;
+ private static PigServer server;
private static String[] input;
private static HiveConf hiveConf;
- private static final String basicFile = "/tmp/basic.input.data";
public void Initialize() throws Exception {
hiveConf = new HiveConf(this.getClass());
@@ -81,8 +79,8 @@ public class TestPassProperties {
String col2 = "b" + i;
input[i] = i + "," + col1 + "," + col2;
}
- MiniCluster.deleteFile(cluster, basicFile);
- MiniCluster.createInputFile(cluster, basicFile, input);
+ HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input);
+ server = new PigServer(ExecType.LOCAL);
}
@Test
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Thu Apr 12 10:08:45 2012
@@ -21,10 +21,10 @@ package org.apache.hcatalog.mapreduce;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
-import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -38,7 +38,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hcatalog.MiniCluster;
+import org.apache.hcatalog.HcatTestUtils;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.common.HCatUtil;
@@ -48,17 +48,16 @@ import org.apache.hcatalog.data.schema.H
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.util.UDFContext;
import org.junit.Test;
public class TestSequenceFileReadWrite {
+ private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
+ "/build/test/data/" + TestSequenceFileReadWrite.class.getCanonicalName();
+ private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
+ private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
- private static MiniCluster cluster = MiniCluster.buildCluster();
private static Driver driver;
- private static Properties props;
private static PigServer server;
- private static final String basicFile = "/tmp/basic.input.data";
- private static String fullFileNameBasic;
private static String[] input;
private static HiveConf hiveConf;
@@ -66,16 +65,12 @@ public class TestSequenceFileReadWrite {
hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname,
- "false");
+ hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
driver = new Driver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
- props = new Properties();
- props.setProperty("fs.default.name", cluster.getProperties()
- .getProperty("fs.default.name"));
- fullFileNameBasic = cluster.getProperties().getProperty(
- "fs.default.name")
- + basicFile;
+
+ new File(TEST_WAREHOUSE_DIR).mkdirs();
int numRows = 3;
input = new String[numRows];
@@ -84,9 +79,8 @@ public class TestSequenceFileReadWrite {
String col2 = "b" + i;
input[i] = i + "," + col1 + "," + col2;
}
- MiniCluster.deleteFile(cluster, basicFile);
- MiniCluster.createInputFile(cluster, basicFile, input);
- server = new PigServer(ExecType.LOCAL, props);
+ HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, input);
+ server = new PigServer(ExecType.LOCAL);
}
@Test
@@ -97,10 +91,9 @@ public class TestSequenceFileReadWrite {
int retCode1 = driver.run(createTable).getResponseCode();
assertTrue(retCode1 == 0);
- UDFContext.getUDFContext().setClientSystemProps();
server.setBatchOn();
server.registerQuery("A = load '"
- + fullFileNameBasic
+ + INPUT_FILE_NAME
+ "' using PigStorage(',') as (a0:int,a1:chararray,a2:chararray);");
server.registerQuery("store A into 'demo_table' using org.apache.hcatalog.pig.HCatStorer();");
server.executeBatch();
@@ -127,10 +120,9 @@ public class TestSequenceFileReadWrite {
int retCode1 = driver.run(createTable).getResponseCode();
assertTrue(retCode1 == 0);
- UDFContext.getUDFContext().setClientSystemProps();
server.setBatchOn();
server.registerQuery("A = load '"
- + fullFileNameBasic
+ + INPUT_FILE_NAME
+ "' using PigStorage(',') as (a0:int,a1:chararray,a2:chararray);");
server.registerQuery("store A into 'demo_table_1' using org.apache.hcatalog.pig.HCatStorer();");
server.executeBatch();
@@ -167,7 +159,7 @@ public class TestSequenceFileReadWrite {
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(DefaultHCatRecord.class);
job.setInputFormatClass(TextInputFormat.class);
- TextInputFormat.setInputPaths(job, this.fullFileNameBasic);
+ TextInputFormat.setInputPaths(job, INPUT_FILE_NAME);
HCatOutputFormat.setOutput(job, OutputJobInfo.create(
MetaStoreUtils.DEFAULT_DATABASE_NAME, "demo_table_2", null));
@@ -178,7 +170,6 @@ public class TestSequenceFileReadWrite {
new FileOutputCommitterContainer(job, null).cleanupJob(job);
assertTrue(job.isSuccessful());
- UDFContext.getUDFContext().setClientSystemProps();
server.setBatchOn();
server.registerQuery("C = load 'default.demo_table_2' using org.apache.hcatalog.pig.HCatLoader();");
server.executeBatch();
@@ -213,7 +204,7 @@ public class TestSequenceFileReadWrite {
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(DefaultHCatRecord.class);
job.setInputFormatClass(TextInputFormat.class);
- TextInputFormat.setInputPaths(job, this.fullFileNameBasic);
+ TextInputFormat.setInputPaths(job, INPUT_FILE_NAME);
HCatOutputFormat.setOutput(job, OutputJobInfo.create(
MetaStoreUtils.DEFAULT_DATABASE_NAME, "demo_table_3", null));
@@ -223,7 +214,6 @@ public class TestSequenceFileReadWrite {
new FileOutputCommitterContainer(job, null).cleanupJob(job);
assertTrue(job.isSuccessful());
- UDFContext.getUDFContext().setClientSystemProps();
server.setBatchOn();
server.registerQuery("D = load 'default.demo_table_3' using org.apache.hcatalog.pig.HCatLoader();");
server.executeBatch();
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoader.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoader.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoader.java Thu Apr 12 10:08:45 2012
@@ -17,47 +17,49 @@
*/
package org.apache.hcatalog.pig;
+import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import junit.framework.TestCase;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hcatalog.MiniCluster;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hcatalog.HcatTestUtils;
import org.apache.hcatalog.data.Pair;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
+import org.apache.pig.ResourceStatistics;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.apache.pig.impl.util.UDFContext;
public class TestHCatLoader extends TestCase {
+ private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
+ "/build/test/data/" + TestHCatLoader.class.getCanonicalName();
+ private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
+ private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
+ private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
private static final String BASIC_TABLE = "junit_unparted_basic";
private static final String COMPLEX_TABLE = "junit_unparted_complex";
private static final String PARTITIONED_TABLE = "junit_parted_basic";
- private static MiniCluster cluster = MiniCluster.buildCluster();
+ private static final String SPECIFIC_SIZE_TABLE = "junit_specific_size";
private static Driver driver;
- private static Properties props;
- private static final String basicFile = "/tmp/basic.input.data";
- private static final String complexFile = "/tmp/complex.input.data";
- private static String fullFileNameBasic;
- private static String fullFileNameComplex;
-
- private static int guardTestCount = 5; // ugh, instantiate using introspection in guardedSetupBeforeClass
+ private static int guardTestCount = 6; // ugh, instantiate using introspection in guardedSetupBeforeClass
private static boolean setupHasRun = false;
private static Map<Integer,Pair<Integer,String>> basicInputData;
@@ -90,16 +92,19 @@ public class TestHCatLoader extends Test
return;
}
+ File f = new File(TEST_WAREHOUSE_DIR);
+ if (f.exists()) {
+ FileUtil.fullyDelete(f);
+ }
+ new File(TEST_WAREHOUSE_DIR).mkdirs();
+
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
driver = new Driver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
- props = new Properties();
- props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
- fullFileNameBasic = cluster.getProperties().getProperty("fs.default.name") + basicFile;
- fullFileNameComplex = cluster.getProperties().getProperty("fs.default.name") + complexFile;
cleanup();
@@ -112,7 +117,7 @@ public class TestHCatLoader extends Test
+ "phnos array<struct<phno:string,type:string>>");
createTable(PARTITIONED_TABLE,"a int, b string","bkt string");
-
+ createTable(SPECIFIC_SIZE_TABLE, "a int, b string");
int LOOP_SIZE = 3;
String[] input = new String[LOOP_SIZE*LOOP_SIZE];
@@ -127,21 +132,20 @@ public class TestHCatLoader extends Test
k++;
}
}
- MiniCluster.createInputFile(cluster, basicFile, input);
-
- MiniCluster.createInputFile(cluster, complexFile,
+ HcatTestUtils.createTestDataFile(BASIC_FILE_NAME, input);
+ HcatTestUtils.createTestDataFile(COMPLEX_FILE_NAME,
new String[]{
- //"Henry Jekyll\t42\t(415-253-6367,hjekyll@contemporary.edu.uk)\t{(PHARMACOLOGY),(PSYCHIATRY)},[PHARMACOLOGY#A-,PSYCHIATRY#B+],{(415-253-6367,cell),(408-253-6367,landline)}",
- //"Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)},[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D],{(415-253-6367,cell),(408-253-6367,landline)}",
+ //"Henry Jekyll\t42\t(415-253-6367,hjekyll@contemporary.edu.uk)\t{(PHARMACOLOGY),(PSYCHIATRY)},[PHARMACOLOGY#A-,PSYCHIATRY#B+],{(415-253-6367,cell),(408-253-6367,landline)}",
+ //"Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)},[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D],{(415-253-6367,cell),(408-253-6367,landline)}",
}
);
- PigServer server = new PigServer(ExecType.LOCAL, props);
- UDFContext.getUDFContext().setClientSystemProps();
+ PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
- server.registerQuery("A = load '"+fullFileNameBasic+"' as (a:int, b:chararray);");
+ server.registerQuery("A = load '"+BASIC_FILE_NAME+"' as (a:int, b:chararray);");
server.registerQuery("store A into '"+BASIC_TABLE+"' using org.apache.hcatalog.pig.HCatStorer();");
+ server.registerQuery("store A into '" + SPECIFIC_SIZE_TABLE + "' using org.apache.hcatalog.pig.HCatStorer();");
server.registerQuery("B = foreach A generate a,b;");
server.registerQuery("B2 = filter B by a < 2;");
server.registerQuery("store B2 into '"+PARTITIONED_TABLE+"' using org.apache.hcatalog.pig.HCatStorer('bkt=0');");
@@ -150,17 +154,16 @@ public class TestHCatLoader extends Test
server.registerQuery("C2 = filter C by a >= 2;");
server.registerQuery("store C2 into '"+PARTITIONED_TABLE+"' using org.apache.hcatalog.pig.HCatStorer('bkt=1');");
- server.registerQuery("D = load '"+fullFileNameComplex+"' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});");
+ server.registerQuery("D = load '"+COMPLEX_FILE_NAME+"' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});");
server.registerQuery("store D into '"+COMPLEX_TABLE+"' using org.apache.hcatalog.pig.HCatStorer();");
server.executeBatch();
}
private void cleanup() throws IOException, CommandNeedRetryException {
- MiniCluster.deleteFile(cluster, basicFile);
- MiniCluster.deleteFile(cluster, complexFile);
dropTable(BASIC_TABLE);
dropTable(COMPLEX_TABLE);
dropTable(PARTITIONED_TABLE);
+ dropTable(SPECIFIC_SIZE_TABLE);
}
protected void guardedTearDownAfterClass() throws Exception {
@@ -183,7 +186,7 @@ public class TestHCatLoader extends Test
public void testSchemaLoadBasic() throws IOException{
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
// test that schema was loaded correctly
server.registerQuery("X = load '"+BASIC_TABLE+"' using org.apache.hcatalog.pig.HCatLoader();");
@@ -198,7 +201,7 @@ public class TestHCatLoader extends Test
}
public void testReadDataBasic() throws IOException {
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
server.registerQuery("X = load '"+BASIC_TABLE+"' using org.apache.hcatalog.pig.HCatLoader();");
Iterator<Tuple> XIter = server.openIterator("X");
@@ -217,22 +220,22 @@ public class TestHCatLoader extends Test
public void testSchemaLoadComplex() throws IOException{
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
// test that schema was loaded correctly
server.registerQuery("K = load '"+COMPLEX_TABLE+"' using org.apache.hcatalog.pig.HCatLoader();");
Schema dumpedKSchema = server.dumpSchema("K");
List<FieldSchema> Kfields = dumpedKSchema.getFields();
- assertEquals(6,Kfields.size());
+ assertEquals(6, Kfields.size());
assertEquals(DataType.CHARARRAY,Kfields.get(0).type);
assertEquals("name",Kfields.get(0).alias.toLowerCase());
assertEquals( DataType.INTEGER,Kfields.get(1).type);
- assertEquals("studentid",Kfields.get(1).alias.toLowerCase());
+ assertEquals("studentid", Kfields.get(1).alias.toLowerCase());
- assertEquals(DataType.TUPLE,Kfields.get(2).type);
- assertEquals("contact",Kfields.get(2).alias.toLowerCase());
+ assertEquals(DataType.TUPLE, Kfields.get(2).type);
+ assertEquals("contact", Kfields.get(2).alias.toLowerCase());
{
assertNotNull(Kfields.get(2).schema);
assertTrue(Kfields.get(2).schema.getFields().size() == 2);
@@ -241,8 +244,8 @@ public class TestHCatLoader extends Test
assertTrue(Kfields.get(2).schema.getFields().get(1).type == DataType.CHARARRAY);
assertTrue(Kfields.get(2).schema.getFields().get(1).alias.equalsIgnoreCase("email"));
}
- assertEquals(DataType.BAG,Kfields.get(3).type);
- assertEquals("currently_registered_courses",Kfields.get(3).alias.toLowerCase());
+ assertEquals(DataType.BAG, Kfields.get(3).type);
+ assertEquals("currently_registered_courses", Kfields.get(3).alias.toLowerCase());
{
assertNotNull(Kfields.get(3).schema);
assertEquals(1,Kfields.get(3).schema.getFields().size());
@@ -257,7 +260,7 @@ public class TestHCatLoader extends Test
assertEquals(DataType.MAP,Kfields.get(4).type);
assertEquals("current_grades",Kfields.get(4).alias.toLowerCase());
assertEquals(DataType.BAG,Kfields.get(5).type);
- assertEquals("phnos",Kfields.get(5).alias.toLowerCase());
+ assertEquals("phnos", Kfields.get(5).alias.toLowerCase());
{
assertNotNull(Kfields.get(5).schema);
assertEquals(1,Kfields.get(5).schema.getFields().size());
@@ -273,12 +276,12 @@ public class TestHCatLoader extends Test
}
public void testReadPartitionedBasic() throws IOException, CommandNeedRetryException {
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
driver.run("select * from "+PARTITIONED_TABLE);
ArrayList<String> valuesReadFromHiveDriver = new ArrayList<String>();
driver.getResults(valuesReadFromHiveDriver);
- assertEquals(basicInputData.size(),valuesReadFromHiveDriver.size());
+ assertEquals(basicInputData.size(), valuesReadFromHiveDriver.size());
server.registerQuery("W = load '"+PARTITIONED_TABLE+"' using org.apache.hcatalog.pig.HCatLoader();");
Schema dumpedWSchema = server.dumpSchema("W");
@@ -337,7 +340,7 @@ public class TestHCatLoader extends Test
public void testProjectionsBasic() throws IOException {
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
// projections are handled by using generate, not "as" on the Load
@@ -379,4 +382,5 @@ public class TestHCatLoader extends Test
}
assertEquals(basicInputData.size(),numTuplesRead);
}
+
}
Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1325183&r1=1325182&r2=1325183&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Apr 12 10:08:45 2012
@@ -50,9 +50,9 @@ import org.junit.Test;
public class TestHCatLoaderComplexSchema {
- private static MiniCluster cluster = MiniCluster.buildCluster();
+ //private static MiniCluster cluster = MiniCluster.buildCluster();
private static Driver driver;
- private static Properties props;
+ //private static Properties props;
private void dropTable(String tablename) throws IOException, CommandNeedRetryException{
driver.run("drop table "+tablename);
@@ -87,8 +87,8 @@ public class TestHCatLoaderComplexSchema
hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
driver = new Driver(hiveConf);
SessionState.start(new CliSessionState(hiveConf));
- props = new Properties();
- props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
+ //props = new Properties();
+ //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
}
@@ -180,7 +180,7 @@ public class TestHCatLoaderComplexSchema
MockLoader.setData(tablename+"Input", data);
try {
createTable(tablename, tableSchema);
- PigServer server = new PigServer(ExecType.LOCAL, props);
+ PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
server.registerQuery("A = load '"+tablename+"Input' using org.apache.hcatalog.pig.MockLoader() AS "+pigSchema+";");
Schema dumpedASchema = server.dumpSchema("A");