You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by an...@apache.org on 2014/03/18 19:36:10 UTC
svn commit: r1579006 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Author: aniket486
Date: Tue Mar 18 18:36:10 2014
New Revision: 1579006
URL: http://svn.apache.org/r1579006
Log:
PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 18 18:36:10 2014
@@ -99,6 +99,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3815: Hadoop bug causes to pig to fail silently with jar cache (aniket486)
+
PIG-3816: Incorrect Javadoc for launchPlan() method (kyungho via prkommireddi)
PIG-3673: Divide by zero error in runpigmix.pl script (suhassatish via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1579006&r1=1579005&r2=1579006&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Mar 18 18:36:10 2014
@@ -20,6 +20,7 @@ package org.apache.pig.backend.hadoop.ex
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.net.URI;
@@ -1639,7 +1640,10 @@ public class JobControlCompiler{
log.info("Found " + url + " in jar cache at "+ stagingDir);
long curTime = System.currentTimeMillis();
fs.setTimes(jarPath, -1, curTime);
- return jarPath;
+ // PIG-3815 In hadoop 1.0, addFileToClassPath uses : as separator
+ // jarPath has full uri at this point, we need to remove hdfs://nn:port
+ // part to avoid parsing errors on backend
+ return new Path(jarPath.toUri().getPath());
}
}
}
@@ -1647,11 +1651,15 @@ public class JobControlCompiler{
// attempt to copy to cache else return null
fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
Path cacheFile = new Path(cacheDir, filename);
- OutputStream os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+ OutputStream os = null;
+ InputStream is = null;
try {
- IOUtils.copyBytes(url.openStream(), os, 4096, true);
+ os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+ is = url.openStream();
+ IOUtils.copyBytes(is, os, 4096, true);
} finally {
- os.close();
+ org.apache.commons.io.IOUtils.closeQuietly(is);
+ org.apache.commons.io.IOUtils.closeQuietly(os);
}
return cacheFile;
@@ -1686,13 +1694,12 @@ public class JobControlCompiler{
Path dst = new Path(FileLocalizer.getTemporaryPath(pigContext).toUri().getPath(), suffix);
FileSystem fs = dst.getFileSystem(conf);
- OutputStream os = fs.create(dst);
+ OutputStream os = null;
try {
+ os = fs.create(dst);
IOUtils.copyBytes(url.openStream(), os, 4096, true);
} finally {
- // IOUtils can not close both the input and the output properly in a finally
- // as we can get an exception in between opening the stream and calling the method
- os.close();
+ org.apache.commons.io.IOUtils.closeQuietly(os);
}
return dst;
}