You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by an...@apache.org on 2014/05/28 21:55:45 UTC
svn commit: r1598116 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Author: aniket486
Date: Wed May 28 19:55:44 2014
New Revision: 1598116
URL: http://svn.apache.org/r1598116
Log:
PIG-3955: Remove url.openStream() file descriptor leak from JCC (aniket486)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1598116&r1=1598115&r2=1598116&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed May 28 19:55:44 2014
@@ -155,6 +155,8 @@ PIG-3882: Multiquery off mode execution
BUG FIXES
+PIG-3955: Remove url.openStream() file descriptor leak from JCC (aniket486)
+
PIG-3958: TestMRJobStats is broken in 0.13 and trunk (aniket486)
PIG-3949: HiveColumnarStorage compile failure with Hive 0.14.0 (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1598116&r1=1598115&r2=1598116&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Wed May 28 19:55:44 2014
@@ -1629,39 +1629,44 @@ public class JobControlCompiler{
private static Path getFromCache(PigContext pigContext,
Configuration conf,
URL url) throws IOException {
+ InputStream is1 = null;
+ InputStream is2 = null;
+ OutputStream os = null;
+
try {
Path stagingDir = getCacheStagingDir(conf);
String filename = FilenameUtils.getName(url.getPath());
- String checksum = DigestUtils.shaHex(url.openStream());
+ is1 = url.openStream();
+ String checksum = DigestUtils.shaHex(is1);
FileSystem fs = FileSystem.get(conf);
Path cacheDir = new Path(stagingDir, checksum);
Path cacheFile = new Path(cacheDir, filename);
if (fs.exists(cacheFile)) {
- log.info("Found " + url + " in jar cache at "+ stagingDir);
- long curTime = System.currentTimeMillis();
- fs.setTimes(cacheFile, -1, curTime);
- return cacheFile;
+ log.debug("Found " + url + " in jar cache at "+ cacheDir);
+ long curTime = System.currentTimeMillis();
+ fs.setTimes(cacheFile, -1, curTime);
+ return cacheFile;
}
- log.info("Url "+ url + " was not found in jarcache at "+ stagingDir);
+ log.info("Url "+ url + " was not found in jarcache at "+ cacheDir);
// attempt to copy to cache else return null
fs.mkdirs(cacheDir, FileLocalizer.OWNER_ONLY_PERMS);
- OutputStream os = null;
- InputStream is = null;
- try {
- os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
- is = url.openStream();
- IOUtils.copyBytes(is, os, 4096, true);
- } finally {
- org.apache.commons.io.IOUtils.closeQuietly(is);
- // IOUtils should not close stream to HDFS quietly
- os.close();
- }
+ is2 = url.openStream();
+ os = FileSystem.create(fs, cacheFile, FileLocalizer.OWNER_ONLY_PERMS);
+ IOUtils.copyBytes(is2, os, 4096, true);
+
return cacheFile;
} catch (IOException ioe) {
log.info("Unable to retrieve jar from jar cache ", ioe);
return null;
+ } finally {
+ org.apache.commons.io.IOUtils.closeQuietly(is1);
+ org.apache.commons.io.IOUtils.closeQuietly(is2);
+ // IOUtils should not close stream to HDFS quietly
+ if (os != null) {
+ os.close();
+ }
}
}