You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tg...@apache.org on 2013/03/26 21:50:47 UTC
svn commit: r1461305 - in
/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src:
main/java/org/apache/hadoop/tools/mapred/
main/java/org/apache/hadoop/tools/util/ test/java/org/apache/hadoop/tools/
Author: tgraves
Date: Tue Mar 26 20:50:46 2013
New Revision: 1461305
URL: http://svn.apache.org/r1461305
Log:
MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream does not close the wrapped InputStream. (Chris Nauroth via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1461305&r1=1461304&r2=1461305&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java Tue Mar 26 20:50:46 2013
@@ -112,7 +112,7 @@ public class RetriableFileCopyCommand ex
tmpTargetPath, true, BUFFER_SIZE,
getReplicationFactor(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath),
getBlockSize(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath), context));
- return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, true, context);
+ return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, context);
}
private void compareFileLengths(FileStatus sourceFileStatus, Path target,
@@ -158,8 +158,8 @@ public class RetriableFileCopyCommand ex
}
private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream,
- int bufferSize, boolean mustCloseStream,
- Mapper.Context context) throws IOException {
+ int bufferSize, Mapper.Context context)
+ throws IOException {
Path source = sourceFileStatus.getPath();
byte buf[] = new byte[bufferSize];
ThrottledInputStream inStream = null;
@@ -175,8 +175,7 @@ public class RetriableFileCopyCommand ex
bytesRead = inStream.read(buf);
}
} finally {
- if (mustCloseStream)
- IOUtils.cleanup(LOG, outStream, inStream);
+ IOUtils.cleanup(LOG, outStream, inStream);
}
return totalBytesRead;
Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java?rev=1461305&r1=1461304&r2=1461305&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java Tue Mar 26 20:50:46 2013
@@ -52,6 +52,11 @@ public class ThrottledInputStream extend
this.maxBytesPerSec = maxBytesPerSec;
}
+ @Override
+ public void close() throws IOException {
+ rawStream.close();
+ }
+
/** @inheritDoc */
@Override
public int read() throws IOException {
Modified: hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java?rev=1461305&r1=1461304&r2=1461305&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java Tue Mar 26 20:50:46 2013
@@ -96,7 +96,7 @@ public class TestIntegration {
try {
addEntries(listFile, "singlefile1/file1");
- createFiles("singlefile1/file1", target.toString());
+ createFiles("singlefile1/file1", "target");
runTest(listFile, target, sync);