You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/10/24 00:11:40 UTC
svn commit: r1535194 -
/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
Author: daijy
Date: Wed Oct 23 22:11:39 2013
New Revision: 1535194
URL: http://svn.apache.org/r1535194
Log:
PIG-3540: e2e test for tez (revert unintended change)
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java?rev=1535194&r1=1535193&r2=1535194&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java Wed Oct 23 22:11:39 2013
@@ -119,22 +119,13 @@ public class TezJobControlCompiler {
private TezJob getJob(TezOperPlan tezPlan) throws JobCreationException {
try {
ApplicationId appId = tezClient.createApplication();
- Map<String, LocalResource> localResources = Maps.newHashMap();
FileSystem remoteFs = FileSystem.get(tezConf);
Path remoteStagingDir = remoteFs.makeQualified(new Path(
tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR), appId.toString()));
-
+
+ // Setup the DistributedCache for this job
for (URL extraJar : pigContext.extraJars) {
- Path pathInHDFS = Utils.shipToHDFS(pigContext, tezConf, extraJar);
- FileStatus fstat = remoteFs.getFileStatus(pathInHDFS);
- LocalResource extraJarRsrc = LocalResource.newInstance(
- ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
- LocalResourceType.FILE,
- LocalResourceVisibility.APPLICATION,
- fstat.getLen(),
- fstat.getModificationTime());
- localResources.put(pathInHDFS.getName(), extraJarRsrc);
- pigContext.skipJars.add(extraJar.getPath());
+ Utils.putJarOnClassPathThroughDistributedCache(pigContext, tezConf, extraJar);
}
// Collect all the UDFs registered in tezPlan
@@ -159,6 +150,7 @@ public class TezJobControlCompiler {
remoteFs.copyFromLocalFile(new Path(jobJar.getAbsolutePath()), remoteJarPath);
FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
+ Map<String, LocalResource> localResources = Maps.newHashMap();
LocalResource dagJarLocalRsrc = LocalResource.newInstance(
ConverterUtils.getYarnUrlFromPath(remoteJarPath),
LocalResourceType.FILE,