You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/10/23 23:49:21 UTC
svn commit: r1535192 - in /pig/branches/tez:
src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
test/e2e/pig/tests/tez.conf
Author: daijy
Date: Wed Oct 23 21:49:21 2013
New Revision: 1535192
URL: http://svn.apache.org/r1535192
Log:
PIG-3540: e2e test for tez
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
pig/branches/tez/test/e2e/pig/tests/tez.conf
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java?rev=1535192&r1=1535191&r2=1535192&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java Wed Oct 23 21:49:21 2013
@@ -119,13 +119,22 @@ public class TezJobControlCompiler {
private TezJob getJob(TezOperPlan tezPlan) throws JobCreationException {
try {
ApplicationId appId = tezClient.createApplication();
+ Map<String, LocalResource> localResources = Maps.newHashMap();
FileSystem remoteFs = FileSystem.get(tezConf);
Path remoteStagingDir = remoteFs.makeQualified(new Path(
tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR), appId.toString()));
-
- // Setup the DistributedCache for this job
+
for (URL extraJar : pigContext.extraJars) {
- Utils.putJarOnClassPathThroughDistributedCache(pigContext, tezConf, extraJar);
+ Path pathInHDFS = Utils.shipToHDFS(pigContext, tezConf, extraJar);
+ FileStatus fstat = remoteFs.getFileStatus(pathInHDFS);
+ LocalResource extraJarRsrc = LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
+ LocalResourceType.FILE,
+ LocalResourceVisibility.APPLICATION,
+ fstat.getLen(),
+ fstat.getModificationTime());
+ localResources.put(pathInHDFS.getName(), extraJarRsrc);
+ pigContext.skipJars.add(extraJar.getPath());
}
// Collect all the UDFs registered in tezPlan
@@ -150,7 +159,6 @@ public class TezJobControlCompiler {
remoteFs.copyFromLocalFile(new Path(jobJar.getAbsolutePath()), remoteJarPath);
FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
- Map<String, LocalResource> localResources = Maps.newHashMap();
LocalResource dagJarLocalRsrc = LocalResource.newInstance(
ConverterUtils.getYarnUrlFromPath(remoteJarPath),
LocalResourceType.FILE,
Modified: pig/branches/tez/test/e2e/pig/tests/tez.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/tests/tez.conf?rev=1535192&r1=1535191&r2=1535192&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/tests/tez.conf (original)
+++ pig/branches/tez/test/e2e/pig/tests/tez.conf Wed Oct 23 21:49:21 2013
@@ -85,90 +85,3 @@ store c into ':OUTPATH:';\,
}
]
}
-#!/usr/bin/env perl
-
-############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-###############################################################################
-# Nightly tests for pig.
-#
-#
-
-#PigSetup::setup();
-
-#my $me = `whoami`;
-#chomp $me;
-
-$cfg = {
- 'driver' => 'Pig',
- 'nummachines' => 5,
-
- 'groups' => [
- {
- 'name' => 'Checkin',
- 'tests' => [
- {
- 'num' => 1,
- 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
-store a into ':OUTPATH:';\,
- },
- {
- 'num' => 2,
- 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k';
-store a into ':OUTPATH:';\,
- },
- {
- 'num' => 3,
- 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
-b = filter a by age>30;
-c = group b by name;
-d = foreach c generate group, COUNT(b) as count;
-e = group d by count;
-store e into ':OUTPATH:';\,
- },
- ]
- },
- {
- 'name' => 'Register',
- 'tests' => [
- {
- 'num' => 1,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
-a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
-b = foreach a generate org.apache.pig.piggybank.evaluation.IsNumeric(name), org.apache.pig.piggybank.evaluation.IsNumeric(age);
-store b into ':OUTPATH:';\,
- },
- ]
- },
- {
- 'name' => 'LoaderStorer',
- 'tests' => [
- {
- 'num' => 1,
- 'pig' => q\register :PIGPATH:/build/ivy/lib/Pig/avro-mapred-1.7.4.jar
-a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);
-store a into ':OUTPATH:.intermediate' using AvroStorage();
-exec
-b = load ':OUTPATH:.intermediate' using AvroStorage();
-c = filter b by age>30;
-store c into ':OUTPATH:';\,
- 'notmq' => 1,
- },
- ]
- }
- ]
- }