You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/09/27 07:49:55 UTC
svn commit: r1526795 - in
/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton: build.xml
conf/default.conf drivers/TestDriverCurl.pm tests/ddl.conf
tests/jobstatus.conf tests/jobsubmission.conf
tests/jobsubmission_streaming.conf
Author: thejas
Date: Fri Sep 27 05:49:55 2013
New Revision: 1526795
URL: http://svn.apache.org/r1526795
Log:
HIVE-5078: [WebHCat] Fix e2e tests on Windows plus test cases for new features (Daniel Dai via Thejas Nair)
Added:
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
Modified:
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/build.xml
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/conf/default.conf
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/ddl.conf
hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
Modified: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/build.xml?rev=1526795&r1=1526794&r2=1526795&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/build.xml Fri Sep 27 05:49:55 2013
@@ -94,6 +94,7 @@
<exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="."/>
+ <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
<env key="TH_WORKING_DIR" value="${test.location}"/>
<env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
<env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
@@ -109,9 +110,10 @@
<env key="SECURE_MODE" value="${secure.mode}"/>
<arg line="${tests.to.run}"/>
<arg value="${basedir}/tests/serverstatus.conf"/>
+ <arg value="${basedir}/tests/jobsubmission_streaming.conf"/>
<arg value="${basedir}/tests/ddl.conf"/>
+ <arg value="${basedir}/tests/jobstatus.conf"/>
<arg value="${basedir}/tests/jobsubmission.conf"/>
- <arg value="${basedir}/tests/jobsubmission2.conf"/>
</exec>
</target>
@@ -124,6 +126,7 @@
<property name="tests.to.run" value=""/>
<exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="${harness.dir}"/>
+ <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
<env key="TH_WORKING_DIR" value="${test.location}"/>
<env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
<env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
Modified: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/conf/default.conf
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/conf/default.conf?rev=1526795&r1=1526794&r2=1526795&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/conf/default.conf (original)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/conf/default.conf Fri Sep 27 05:49:55 2013
@@ -1,3 +1,4 @@
+############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+
my $me = `whoami`;
chomp $me;
Modified: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1526795&r1=1526794&r2=1526795&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Fri Sep 27 05:49:55 2013
@@ -1,3 +1,4 @@
+############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+
package TestDriverCurl;
###########################################################################
@@ -35,6 +36,7 @@ use strict;
use English;
use Storable qw(dclone);
use File::Glob ':glob';
+use JSON::Path;
my $passedStr = 'passed';
my $failedStr = 'failed';
@@ -150,13 +152,16 @@ sub new
sub globalSetup
{
my ($self, $globalHash, $log) = @_;
+ my $subName = (caller(0))[3];
+
# Setup the output path
my $me = `whoami`;
chomp $me;
- my $jobId = $globalHash->{'job-id'};
- my $timeId = time;
- $globalHash->{'runid'} = $me . "-" . $timeId . "-" . $jobId;
+ #usernames on windows can be "domain\username" change the "\"
+ # as runid is used in file names
+ $me =~ s/\\/_/;
+ $globalHash->{'runid'} = $me . "." . time;
# if "-ignore false" was provided on the command line,
# it means do run tests even when marked as 'ignore'
@@ -170,7 +175,6 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
- $globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'};
$globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'};
$globalHash->{'current_user'} = $ENV{'USER_NAME'};
@@ -186,11 +190,6 @@ sub globalSetup
$globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'};
$globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'};
- }
-
-sub globalSetupConditional
- {
- my ($self, $globalHash, $log) = @_;
# add libexec location to the path
if (defined($ENV{'PATH'})) {
@@ -225,12 +224,6 @@ sub globalSetupConditional
# None
sub globalCleanup
{
- # noop there because the removal of temp directories, which are created in #globalSetupConditional(), is to be
- # performed in method #globalCleanupConditional().
- }
-
-sub globalCleanupConditional
- {
my ($self, $globalHash, $log) = @_;
IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
@@ -292,6 +285,17 @@ sub replaceParameters
my @new_options = ();
foreach my $option (@options) {
$option = $self->replaceParametersInArg($option, $testCmd, $log);
+ if (isWindows()) {
+ my $equal_pos = index($option, '=');
+ if ($equal_pos != -1) {
+ my $left = substr($option, 0, $equal_pos);
+ my $right = substr($option, $equal_pos+1);
+ if ($right =~ /=/) {
+ $right = '"'.$right.'"';
+ $option = $left . "=" . $right;
+ }
+ }
+ }
push @new_options, ($option);
}
$testCmd->{$aPfix . 'post_options'} = \@new_options;
@@ -306,6 +310,15 @@ sub replaceParameters
}
}
+ if (defined $testCmd->{$aPfix . 'json_path'}) {
+ my $json_path_matches = $testCmd->{$aPfix . 'json_path'};
+ my @keys = keys %{$json_path_matches};
+
+ foreach my $key (@keys) {
+ my $new_value = $self->replaceParametersInArg($json_path_matches->{$key}, $testCmd, $log);
+ $json_path_matches->{$key} = $new_value;
+ }
+ }
}
@@ -498,7 +511,7 @@ sub execCurlCmd(){
$testCmd->{'http_daemon'} = $d;
$testCmd->{'callback_url'} = $d->url . 'templeton/$jobId';
push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'});
- # push ${testCmd->{'post_options'}}, ('callback=' . $testCmd->{'callback_url'});
+ push @{$testCmd->{$argPrefix . 'post_options'}}, ('callback=' . $testCmd->{'callback_url'});
# #my @options = @{$testCmd->{'post_options'}};
# print $log "post options @options\n";
}
@@ -510,7 +523,7 @@ sub execCurlCmd(){
push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header);
push @curl_cmd, ($url);
- print $log "$0:$subName Going to run command : " . join (' ', @curl_cmd);
+ print $log "$0:$subName Going to run command : " . join (' , ', @curl_cmd);
print $log "\n";
@@ -604,6 +617,37 @@ sub compare
my $json_hash;
my %json_info;
+ # for information on JSONPath, check http://goessner.net/articles/JsonPath/
+ if (defined $testCmd->{'json_path'}) {
+ my $json_matches = $testCmd->{'json_path'};
+ foreach my $key (keys %$json_matches) {
+ my $regex_expected_value = $json_matches->{$key};
+ my $path = JSON::Path->new($key);
+ my $value;
+ # when filter_job_status is defined
+ if (defined $testCmd->{'filter_job_status'}) {
+ # decode $testResult->{'body'} to an array of hash
+ my $body = decode_json $testResult->{'body'};
+ # in the tests, we run this case with jobName = "PigLatin:loadstore.pig"
+ # filter $body to leave only records with this jobName
+ my @filtered_body = grep {($_->{detail}{profile}{jobName} eq "PigLatin:loadstore.pig")} @$body;
+ my @sorted_filtered_body = sort { $a->{id} <=> $b->{id} } @filtered_body;
+ $value = $path->value(\@sorted_filtered_body);
+ } else {
+ $value = $path->value($testResult->{'body'});
+ }
+
+ if ($value !~ /$regex_expected_value/s) {
+ print $log "$0::$subName INFO check failed:"
+ . " json pattern check failed. For field "
+ . "$key, regex <" . $regex_expected_value
+ . "> did not match the result <" . $value
+ . ">\n";
+ $result = 0;
+ last;
+ }
+ }
+ }
if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) {
my $json = new JSON;
$json_hash = $json->utf8->decode($testResult->{'body'});
@@ -639,7 +683,7 @@ sub compare
print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n";
if ($json_field_val !~ /$regex_expected_value/s) {
- print $log "$0::$subName INFO check failed:"
+ print $log "$0::$subName WARN check failed:"
. " json pattern check failed. For field "
. "$key, regex <" . $regex_expected_value
. "> did not match the result <" . $json_field_val
@@ -654,7 +698,7 @@ sub compare
print $log "Comparing $key: " . dump($json_field_val) . ",expected value: " . dump($regex_expected_obj);
if (!Compare($json_field_val, $regex_expected_obj)) {
- print $log "$0::$subName INFO check failed:"
+ print $log "$0::$subName WARN check failed:"
. " json compare failed. For field "
. "$key, regex <" . dump($regex_expected_obj)
. "> did not match the result <" . dump($json_field_val)
@@ -671,7 +715,7 @@ sub compare
sleep $testCmd->{'kill_job_timeout'};
my $jobid = $json_hash->{'id'};
if (!defined $jobid) {
- print $log "$0::$subName INFO check failed: "
+ print $log "$0::$subName WARN check failed: "
. "no jobid (id field)found in result";
$result = 0;
} else {
@@ -682,13 +726,14 @@ sub compare
#try to get the call back url request until timeout
if ($result == 1 && defined $testCmd->{'check_call_back'}) {
- my $d = $testCmd->{'http_daemon'};
- if (defined $testCmd->{'timeout_seconds'}) {
- $d->timeout($testCmd->{'timeout_seconds'})
- }
- else {
- $d->timeout(300); #wait for 5 mins by default
+
+ my $timeout = 300; #wait for 5 mins for callback
+ if(defined $testCmd->{'timeout'}){
+ $timeout = $testCmd->{'timeout'};
}
+
+ my $d = $testCmd->{'http_daemon'};
+ $d->timeout($timeout);
my $url_requested;
$testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g;
print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n";
@@ -717,13 +762,12 @@ sub compare
}
-
if ( (defined $testCmd->{'check_job_created'})
|| (defined $testCmd->{'check_job_complete'})
- || (defined $testCmd->{'check_job_exit_value'}) ) {
+ || (defined $testCmd->{'check_job_exit_value'}) ) {
my $jobid = $json_hash->{'id'};
if (!defined $jobid) {
- print $log "$0::$subName INFO check failed: "
+ print $log "$0::$subName WARN check failed: "
. "no jobid (id field)found in result";
$result = 0;
} else {
@@ -731,7 +775,7 @@ sub compare
my $json = new JSON;
my $res_hash = $json->utf8->decode($jobResult->{'body'});
if (! defined $res_hash->{'status'}) {
- print $log "$0::$subName INFO check failed: "
+ print $log "$0::$subName WARN check failed: "
. "jobresult not defined ";
$result = 0;
}
@@ -739,10 +783,6 @@ sub compare
my $jobComplete;
my $NUM_RETRIES = 60;
my $SLEEP_BETWEEN_RETRIES = 5;
- if (defined $testCmd->{'timeout_seconds'} && $testCmd->{'timeout_seconds'} > 0) {
- $SLEEP_BETWEEN_RETRIES = ($testCmd->{'timeout_seconds'} / $NUM_RETRIES);
- print $log "found timeout_seconds & set SLEEP_BETWEEN_RETRIES=$SLEEP_BETWEEN_RETRIES";
- }
#first wait for job completion
while ($NUM_RETRIES-- > 0) {
@@ -756,7 +796,7 @@ sub compare
$res_hash = $json->utf8->decode($jobResult->{'body'});
}
if ( (!defined $jobComplete) || lc($jobComplete) ne "true") {
- print $log "$0::$subName INFO check failed: "
+ print $log "$0::$subName WARN check failed: "
. " timeout on wait for job completion ";
$result = 0;
} else {
@@ -772,12 +812,140 @@ sub compare
if (defined($testCmd->{'check_job_exit_value'})) {
my $exitValue = $res_hash->{'exitValue'};
my $expectedExitValue = $testCmd->{'check_job_exit_value'};
- if ( (!defined $exitValue) || $exitValue ne $expectedExitValue) {
+ if ( (!defined $exitValue) || $exitValue % 128 ne $expectedExitValue) {
print $log "check_job_exit_value failed. got exitValue $exitValue, expected $expectedExitValue";
$result = 0;
}
}
}
+
+ #Check userargs
+ print $log "$0::$subName INFO Checking userargs";
+ my @options = @{$testCmd->{'post_options'}};
+ if( !defined $res_hash->{'userargs'}){
+ print $log "$0::$subName INFO expected userargs"
+ . " but userargs not defined\n";
+ $result = 0;
+ }
+
+ #create exp_userargs hash from @options
+ my %exp_userargs = ();
+ foreach my $opt ( @options ){
+ print $log "opt $opt";
+ my ($key, $val) = split q:=:, $opt, 2;
+ if(defined $exp_userargs{$key}){
+
+ #if we have already seen this value
+ #then make the value an array and push new value in
+ if(ref($exp_userargs{$key}) eq ""){
+ my @ar = ($exp_userargs{$key});
+ $exp_userargs{$key} = \@ar;
+ }
+ my $ar = $exp_userargs{$key};
+ push @$ar, ($val);
+ }
+ else{
+ $exp_userargs{$key} = $val;
+ }
+ }
+
+ my %r_userargs = %{$res_hash->{'userargs'}};
+ foreach my $key( keys %exp_userargs){
+ if( !defined $r_userargs{$key}){
+ print $log "$0::$subName INFO $key not found in userargs \n";
+ $result = 0;
+ next;
+ }
+
+ print $log "$0::$subName DEBUG comparing expected "
+ . " $key ->" . dump($exp_userargs{$key})
+ . " With result $key ->" . dump($r_userargs{$key}) . "\n";
+
+ if (!Compare($exp_userargs{$key}, $r_userargs{$key})) {
+ print $log "$0::$subName WARN check failed:"
+ . " json compare failed. For field "
+ . "$key, regex <" . dump($r_userargs{$key})
+ . "> did not match the result <" . dump($exp_userargs{$key})
+ . ">\n";
+ $result = 0;
+ }
+ }
+ if ($result != 0 && $testCmd->{'check_logs'}) {
+ my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+ $testCmdBasics->{'method'} = 'GET';
+ $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs?op=LISTSTATUS';
+ my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log);
+ my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix");
+ my @value = $path->values($curl_result->{'body'});
+ if ($testCmd->{'check_logs'}->{'job_num'} && $testCmd->{'check_logs'}->{'job_num'} ne (scalar @value)-1) {
+ print $log "$0::$subName INFO check failed: "
+ . " Expect " . $testCmd->{'check_logs'}->{'job_num'} . " jobs in logs, but get " . scalar @value;
+ $result = 0;
+ return $result;
+ }
+ foreach my $jobid (@value) {
+ if ($jobid eq 'list.txt') {
+ next;
+ }
+ my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+ $testCmdBasics->{'method'} = 'GET';
+ $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '?op=LISTSTATUS';
+ my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log);
+
+ my $path = JSON::Path->new("FileStatuses.FileStatus[*]");
+ my @value = $path->values($curl_result->{'body'});
+
+ my $foundjobconf = 0;
+ foreach my $elem (@value) {
+ if ($elem->{'pathSuffix'} eq "job.xml.html") {
+ $foundjobconf = 1;
+ if ($elem->{'length'} eq "0") {
+ print $log "$0::$subName INFO check failed: "
+ . " job.xml.html for " . $jobid . " is empty";
+ $result = 0;
+ return $result;
+ }
+ next;
+ }
+ my $attempt = $elem->{'pathSuffix'};
+ my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+ $testCmdBasics->{'method'} = 'GET';
+ $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1:OUTDIR:' . '/status/logs/' . $jobid . '/' . $attempt . '?op=LISTSTATUS';
+ my $curl_result = $self->execCurlCmd($testCmdBasics, "", $log);
+ my $path = JSON::Path->new("FileStatuses.FileStatus[*].pathSuffix");
+ my @value = $path->values($curl_result->{'body'});
+ my @files = ('stderr', 'stdout', 'syslog');
+ foreach my $file (@files) {
+ if ( !grep( /$file/, @value ) ) {
+ print $log "$0::$subName INFO check failed: "
+ . " Cannot find " . $file . " in logs/" . $attempt;
+ $result = 0;
+ return $result;
+ }
+ }
+ $path = JSON::Path->new("FileStatuses.FileStatus[*].length");
+ @value = $path->values($curl_result->{'body'});
+ my $foundnonzerofile = 0;
+ foreach my $length (@value) {
+ if ($length ne "0") {
+ $foundnonzerofile = 1;
+ }
+ }
+ if (!$foundnonzerofile) {
+ print $log "$0::$subName INFO check failed: "
+ . " All files in logs/" . $attempt . " are empty";
+ $result = 0;
+ return $result;
+ }
+ }
+ if (!$foundjobconf) {
+ print $log "$0::$subName INFO check failed: "
+ . " Cannot find job.xml.html for " . $jobid;
+ $result = 0;
+ return $result;
+ }
+ }
+ }
}
}
}
@@ -1357,5 +1525,13 @@ sub tmpIPCRunJoinStdoe {
return ( $? );
}
-
+sub isWindows
+{
+ if($^O =~ /mswin/i) {
+ return 1;
+ }
+ else {
+ return 0;
+ }
+}
1;
Modified: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/ddl.conf
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/ddl.conf?rev=1526795&r1=1526794&r2=1526795&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/ddl.conf (original)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/ddl.conf Fri Sep 27 05:49:55 2013
@@ -284,7 +284,7 @@ $cfg =
# 'json_field_substr_match' => {'table-name' => 'templeton_testtab1'},
'json_field_match_object' => { 'columns' => '[
{ "name" : "i", "type" : "int", "comment" : "from deserializer" },
- { "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
+ { "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
]' },
},
{ #drop table
@@ -354,7 +354,7 @@ $cfg =
"fieldsTerminatedBy" : "\u0001",
"collectionItemsTerminatedBy" : "\u0002",
"mapKeysTerminatedBy" : "\u0003",
- "linesTerminatedBy" : "\n",
+ "linesTerminatedBy" : "\\\n",
"serde" : {
"name" : "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe",
@@ -406,6 +406,7 @@ $cfg =
"type" : "bigint", "comment" : "from deserializer"
},
{
+ "comment" : "IP Address of the User",
"name" : "ip",
"type" : "string", "comment" : "from deserializer"
}
@@ -516,15 +517,7 @@ $cfg =
'url' => ':TEMPLETON_URL:/templeton/v1/ddl',
'status_code' => 200,
'post_options' => ['user.name=:UNAME:',
- 'exec=create table if not exists templetontest_parts (i int, j bigint, ip STRING COMMENT "IP Address of the User")
-COMMENT "This is the page view table"
- PARTITIONED BY(dt STRING, country STRING)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY "\001"
- COLLECTION ITEMS TERMINATED BY "\002"
- MAP KEYS TERMINATED BY "\003"
-STORED AS rcfile
---LOCATION "table1_location" '],
+ 'exec=create table if not exists templetontest_parts (i int, j bigint, ip STRING COMMENT \'IP Address of the User\') COMMENT \'This is the page view table\' PARTITIONED BY(dt STRING, country STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY \'\001\' COLLECTION ITEMS TERMINATED BY \'\002\' MAP KEYS TERMINATED BY \'\003\' STORED AS rcfile --LOCATION \'table1_location\' '],
'json_field_substr_match' => {'stderr' => 'OK'},
},
{
@@ -632,6 +625,7 @@ STORED AS rcfile
"type" : "bigint", "comment" : "from deserializer"
},
{
+ "comment" : "IP Address of the User",
"name" : "ip",
"type" : "string", "comment" : "from deserializer"
}
@@ -684,6 +678,7 @@ STORED AS rcfile
"type" : "bigint", "comment" : "from deserializer"
},
{
+ "comment" : "IP Address of the User",
"name" : "ip",
"type" : "string", "comment" : "from deserializer"
}
@@ -732,7 +727,7 @@ STORED AS rcfile
'status_code' => 404,
'json_field_substr_match' =>
{
- 'error' => 'FAILED: SemanticException \[Error 10006\]: Partition not found \{dt=20120101\, country=IN\}'
+ 'error' => 'Partition not found {dt=20120101, country=IN}'
},
},
@@ -761,7 +756,7 @@ STORED AS rcfile
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/ddl?user.name=:UNAME:',
'status_code' => 200,
- 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testcol_tab (i int comment "column with comment", j bigint) STORED AS rcfile;'],
+ 'post_options' => ['user.name=:UNAME:','exec=create table if not exists templeton_testcol_tab (i int comment \'column with comment\', j bigint) STORED AS rcfile;'],
'json_field_substr_match' => {'stderr' => 'OK'},
},
{
@@ -779,7 +774,7 @@ STORED AS rcfile
'json_field_match_object' =>
{
'columns' => '[
- { "name" : "i", "type" : "int", "comment" : "from deserializer"},
+ { "name" : "i", "type" : "int", "comment" : "from deserializer" },
{ "name" : "j", "type" : "bigint", "comment" : "from deserializer" }
]'
},
@@ -1088,9 +1083,7 @@ STORED AS rcfile
'status_code' => 200,
'post_options' => ['user.name=:UNAME:',
'permissions=---------',
- 'exec=create table templetontest_hcatgp(i int, j bigint)
- PARTITIONED BY(dt STRING, country STRING)
- STORED AS rcfile;'
+ 'exec=create table templetontest_hcatgp(i int, j bigint) PARTITIONED BY(dt STRING, 3Bcountry STRING) STORED AS rcfile;'
],
'json_field_substr_match' => {'stderr' => 'OK', 'exitcode' => '^0$'}
},
Added: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf?rev=1526795&view=auto
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (added)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf Fri Sep 27 05:49:55 2013
@@ -0,0 +1,160 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+
+##=============================================================================================================
+ {
+ 'name' => 'JOBS',
+ 'tests' =>
+ [
+ {
+ #a simple load store script as UNAME_OTHER
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #a simple load store script as UNAME
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ #a simple load store script as UNAME_OTHER
+ 'num' => 3,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME_OTHER:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_call_back' => 1,
+ },
+ {
+ # now we have 6 jobs (two of each previous test case), check if they have the right user tag
+ # GET jobs?user.name=UNAME_OTHER&showall=true&fields=*, should get all 6 jobs, with the right username field
+ 'num' => 4,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&showall=true&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME:',
+ '$[-4:].detail.status.username' => ':UNAME:', '$[-5:].detail.status.username' => ':UNAME_OTHER:', '$[-6:].detail.status.username' => ':UNAME_OTHER:'},
+ 'status_code' => 200,
+ 'filter_job_status' => 1,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER&fields=*, should get only jobs launched as UNAME_OTHER
+ 'num' => 5,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].detail.status.username' => ':UNAME_OTHER:', '$[-2:].detail.status.username' => ':UNAME_OTHER:', '$[-3:].detail.status.username' => ':UNAME_OTHER:',
+ '$[-4:].detail.status.username' => ':UNAME_OTHER:'},
+ 'status_code' => 200,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER, only get jobid but no detail
+ 'num' => 6,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].id' => 'job_.*', '$[-1:].detail' => '^$'},
+ 'status_code' => 200,
+ },
+ {
+ # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the job
+ 'num' => 7,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:].id' => 'job_.*',
+ '$[-1:].detail.status.jobId' => 'job_.*',
+ '$[-1:].detail.status.runState' => '\\d+',
+ '$[-1:].detail.status.jobId' => 'job_.*',
+ '$[-1:].detail.status.jobComplete' => 'true',
+ '$[-1:].detail.profile.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.profile.jobFile' => '^.+$',
+ '$[-1:].detail.profile.url' => '^.+$',
+ '$[-1:].detail.profile.queueName' => '^.+$',
+ '$[-1:].detail.profile.jobName' => 'loadstore\.pig',
+ '$[-1:].detail.profile.jobID.id' => '\\d+',
+ '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+',
+ '$[-1:].detail.profile.jobId' => 'job_.*',
+ '$[-1:].detail.id' => 'job_.*',
+ '$[-1:].detail.parentId' => 'job_.*',
+ '$[-1:].detail.percentComplete' => '100%',
+ '$[-1:].detail.exitValue' => '0',
+ '$[-1:].detail.user' => ':UNAME_OTHER:',
+ '$[-1:].detail.callback' => '^.+$',
+ '$[-1:].detail.completed' => 'done',
+ },
+ 'status_code' => 200,
+ },
+ {
+ # GET queue?user.name=UNAME_OTHER, only get jobid as an array
+ 'num' => 8,
+ 'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+ 'method' => 'GET',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/queue?user.name=:UNAME_OTHER:',
+ 'format_header' => 'Content-Type: application/json',
+ 'json_path' => {'$[-1:]' => 'job_.*'},
+ 'status_code' => 200,
+ },
+
+ ]
+ }
+ ]
+}
+;
Modified: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf?rev=1526795&r1=1526794&r2=1526795&view=diff
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf (original)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission.conf Fri Sep 27 05:49:55 2013
@@ -35,79 +35,76 @@ $cfg =
[
##=============================================================================================================
{
- 'name' => 'TestStreaming',
+ 'name' => 'TestKillJob',
'tests' =>
[
{
'num' => 1,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
- 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/cat', 'reducer=/usr/bin/wc'],
- 'json_field_substr_match' => { 'id' => '\d+'},
- #results
- 'status_code' => 200,
- 'check_job_created' => 1,
- 'check_job_complete' => 'SUCCESS',
- 'check_job_exit_value' => 0,
- 'check_call_back' => 1,
- },
- {
- #-ve test - no input file
- 'num' => 2,
- 'ignore' => 'wait for fix in hadoop 1.0.3',
- 'method' => 'POST',
- 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
- 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/ls no_such-file-12e3', 'reducer=/usr/bin/wc'],
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt',
+ 'input=:INPDIR_HDFS:/nums.txt',
+ 'output=:OUTDIR:/mycounts',
+ 'mapper=sleep 100', 'reducer=wc'],
'json_field_substr_match' => { 'id' => '\d+'},
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
- 'check_call_back' => 1,
+ 'check_job_complete' => 'KILLED',
+# 'check_call_back' => 1, #TODO - enable call back check after fix
+ 'kill_job_timeout' => 10,
},
-
]
},
##=============================================================================================================
{
- 'name' => 'TestKillJob',
+ 'name' => 'TestMapReduce',
'tests' =>
[
{
+
'num' => 1,
'method' => 'POST',
- 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
- 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
- 'mapper=/bin/sleep 100', 'reducer=/usr/bin/wc'],
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
+ 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt',
+ 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', ],
'json_field_substr_match' => { 'id' => '\d+'},
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'KILLED',
-# 'check_call_back' => 1, #TODO - enable call back check after fix
- 'kill_job_timeout' => 10,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
},
- ]
- },
-##=============================================================================================================
- {
- 'name' => 'TestMapReduce',
- 'tests' =>
- [
{
- 'num' => 1,
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
+ 'post_options' => ['user.name=:UNAME:','arg=-mt', 'arg=660000',
+ 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=sleep', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ 'timeout' => 840, #increase timeout as this test takes long
+ },
+ {
+ # with log enabled
+ 'num' => 3,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar',
'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt',
- 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', ],
+ 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', 'statusdir=:OUTDIR:/status', 'enablelog=true' ],
'json_field_substr_match' => { 'id' => '\d+'},
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'SUCCESS',
+ 'check_job_complete' => 'SUCCESS',
+ 'check_logs' => { 'job_num' => '1' },
'check_job_exit_value' => 0,
'check_call_back' => 1,
},
@@ -120,7 +117,6 @@ $cfg =
[
{
#test syntax error
- 'ignore' => 'fails in current version',
'num' => 1,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/pig',
@@ -129,7 +125,6 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
'check_job_exit_value' => 8,
'check_call_back' => 1,
},
@@ -143,8 +138,7 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_exit_value' => 0,
- 'check_job_complete' => 'SUCCESS',
+ 'check_job_complete' => 'SUCCESS',
'check_job_exit_value' => 0,
'check_call_back' => 1,
},
@@ -158,7 +152,7 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'SUCCESS',
+ 'check_job_complete' => 'SUCCESS',
'check_job_exit_value' => 0,
'check_call_back' => 1,
},
@@ -228,21 +222,50 @@ $cfg =
{
#no file to be copied, should result in launcher job error
'num' => 8,
- ignore => 'check is disabled for now in templeton',
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/pig',
- 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg= OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/no_such_file.pig',
+ 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/no_such_file.pig',
'files=:INPDIR_HDFS:/rowcountmacro.pig' ],
'json_field_substr_match' => { 'error' => 'does not exist'},
#results
- 'status_code' => 200,
- 'check_job_complete' => 'FAILURE',
+ 'status_code' => 400,
},
+ {
+ #Auto add quote around args
+ 'ignore' => 'MS9 feature, will reenable later',
+ 'num' => 9,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:','arg=-check', 'file=:INPDIR_HDFS:/loadstore.pig', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', ],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ {
+ #a simple load store script with log enabled
+ 'num' => 9,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/pig',
+ 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig',
+ 'statusdir=:OUTDIR:/status', 'enablelog=true'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_logs' => { 'job_num' => '1' },
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
- #test 9
+ #test 10
#TODO jython test
@@ -257,7 +280,6 @@ $cfg =
[
{
#test syntax error
- 'ignore' => 'fails in current version',
'num' => 1,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
@@ -266,8 +288,7 @@ $cfg =
#results
'status_code' => 200,
'check_job_created' => 1,
- 'check_job_complete' => 'FAILURE',
- 'check_job_exit_value' => 11,
+ 'check_job_exit_value' => 64,
},
@@ -305,7 +326,7 @@ $cfg =
'num' => 4,
'method' => 'POST',
'url' => ':TEMPLETON_URL:/templeton/v1/hive',
- 'post_options' => ['user.name=:UNAME:','execute=create external table mynums(a int, b int) location ":INPDIR_HDFS:/numstable/";', ],
+ 'post_options' => ['user.name=:UNAME:','execute=create external table mynums(a int, b int) location \':INPDIR_HDFS:/numstable/\';', ],
'json_field_substr_match' => { 'id' => '\d+'},
#results
'status_code' => 200,
@@ -412,7 +433,49 @@ $cfg =
]
},
-
+ {
+ #test add jar
+ 'num' => 9,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ {
+ #test add jar when the jar is not shipped
+ 'num' => 10,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 1,
+ 'check_call_back' => 1,
+ },
+ {
+ #enable logs
+ 'num' => 11,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=select a,b from mynums', 'statusdir=:OUTDIR:/status', 'enablelog=true'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_logs' => { 'job_num' => '1' },
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
Added: hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf?rev=1526795&view=auto
==============================================================================
--- hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf (added)
+++ hive/branches/branch-0.12/hcatalog/src/test/e2e/templeton/tests/jobsubmission_streaming.conf Fri Sep 27 05:49:55 2013
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'TestStreaming',
+ 'tests' =>
+ [
+ {
+ 'num' => 1,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=cat', 'reducer=wc'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ {
+ #-ve test - no input file
+ 'num' => 2,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt','input=:INPDIR_HDFS:/nums.txt','output=:OUTDIR:/mycounts',
+ 'mapper=ls no_such-file-12e3', 'reducer=wc'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'FAILURE',
+ 'check_job_exit_value' => 1,
+ 'check_call_back' => 1,
+ },
+ {
+ #enable log
+ 'num' => 3,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:',
+ 'input=:INPDIR_HDFS:/nums.txt','input=:INPDIR_HDFS:/nums.txt',
+ 'output=:OUTDIR:/mycounts',
+ 'mapper=cat', 'reducer=wc', 'statusdir=:OUTDIR:/status', 'enablelog=true'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'SUCCESS',
+ 'check_logs' => { 'job_num' => '1' },
+ 'check_job_exit_value' => 0,
+ 'check_call_back' => 1,
+ },
+ {
+ #enable log, failed job case
+ 'num' => 4,
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming',
+ 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt',
+ 'input=:INPDIR_HDFS:/nums.txt',
+ 'output=:OUTDIR:/mycounts',
+ 'mapper=ls no_such-file-12e3', 'reducer=wc', 'statusdir=:OUTDIR:/status', 'enablelog=true'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'FAILURE',
+ 'check_logs' => { 'job_num' => '1' },
+ 'check_job_exit_value' => 1,
+ 'check_call_back' => 1,
+ },
+ ]
+ },
+##=============================================================================================================
+
+
+ ]
+},
+ ;
+