svn commit: r1579043 - /pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java

[an...@apache.org]

Author: aniket486
Date: Tue Mar 18 21:23:40 2014
New Revision: 1579043

URL: http://svn.apache.org/r1579043
Log:
PIG-3815 Hadoop bug causes to pig to fail silently with jar cache - fix hdfs stream close

Modified:
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1579043&r1=1579042&r2=1579043&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Mar 18 21:23:40 2014
@@ -1640,7 +1640,7 @@ public class JobControlCompiler{
                         log.info("Found " + url + " in jar cache at "+ stagingDir);
                         long curTime = System.currentTimeMillis();
                         fs.setTimes(jarPath, -1, curTime);
-                        // PIG-3815 In hadoop 1.0, addFileToClassPath uses : as separator
+                        // PIG-3815 In hadoop 0.20, addFileToClassPath uses : as separator
                         // jarPath has full uri at this point, we need to remove hdfs://nn:port
                         // part to avoid parsing errors on backend
                         return new Path(jarPath.toUri().getPath());
@@ -1659,7 +1659,8 @@ public class JobControlCompiler{
                 IOUtils.copyBytes(is, os, 4096, true);
             } finally {
                 org.apache.commons.io.IOUtils.closeQuietly(is);
-                org.apache.commons.io.IOUtils.closeQuietly(os);
+                // IOUtils should not close stream to HDFS quietly
+                os.close();
             }
             return cacheFile;
 
@@ -1695,11 +1696,15 @@ public class JobControlCompiler{
         Path dst = new Path(FileLocalizer.getTemporaryPath(pigContext).toUri().getPath(), suffix);
         FileSystem fs = dst.getFileSystem(conf);
         OutputStream os = null;
+        InputStream is = null;
         try {
+            is = url.openStream();
             os = fs.create(dst);
-            IOUtils.copyBytes(url.openStream(), os, 4096, true);
+            IOUtils.copyBytes(is, os, 4096, true);
         } finally {
-            org.apache.commons.io.IOUtils.closeQuietly(os);
+            org.apache.commons.io.IOUtils.closeQuietly(is);
+            // IOUtils should not close stream to HDFS quietly
+            os.close();
         }
         return dst;
     }