You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/12/06 07:57:18 UTC
svn commit: r1548386 - in /hadoop/common/branches/HDFS-4685: ./
hadoop-project/ hadoop-tools/hadoop-distcp/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/
hadoo...
Author: cnauroth
Date: Fri Dec 6 06:57:15 2013
New Revision: 1548386
URL: http://svn.apache.org/r1548386
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java
- copied unchanged from r1548385, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestRetriableFileCopyCommand.java
Modified:
hadoop/common/branches/HDFS-4685/ (props changed)
hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
Propchange: hadoop/common/branches/HDFS-4685/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2:r1547755
Merged /hadoop/common/trunk:r1547224-1548385
Modified: hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml?rev=1548386&r1=1548385&r2=1548386&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-project/pom.xml Fri Dec 6 06:57:15 2013
@@ -467,7 +467,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.1</version>
+ <version>2.4</version>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/pom.xml?rev=1548386&r1=1548385&r2=1548386&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/pom.xml Fri Dec 6 06:57:15 2013
@@ -95,6 +95,11 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java?rev=1548386&r1=1548385&r2=1548386&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java Fri Dec 6 06:57:15 2013
@@ -32,6 +32,8 @@ import org.apache.hadoop.tools.util.Dist
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.Credentials;
+import com.google.common.annotations.VisibleForTesting;
+
import java.io.*;
import java.util.Stack;
@@ -107,12 +109,13 @@ public class SimpleCopyListing extends C
/** {@inheritDoc} */
@Override
public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException {
-
- SequenceFile.Writer fileListWriter = null;
-
+ doBuildListing(getWriter(pathToListingFile), options);
+ }
+
+ @VisibleForTesting
+ public void doBuildListing(SequenceFile.Writer fileListWriter,
+ DistCpOptions options) throws IOException {
try {
- fileListWriter = getWriter(pathToListingFile);
-
for (Path path: options.getSourcePaths()) {
FileSystem sourceFS = path.getFileSystem(getConf());
path = makeQualified(path);
@@ -143,8 +146,10 @@ public class SimpleCopyListing extends C
localFile, options);
}
}
+ fileListWriter.close();
+ fileListWriter = null;
} finally {
- IOUtils.closeStream(fileListWriter);
+ IOUtils.cleanup(LOG, fileListWriter);
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1548386&r1=1548385&r2=1548386&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java Fri Dec 6 06:57:15 2013
@@ -30,6 +30,8 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.google.common.annotations.VisibleForTesting;
+
import java.io.*;
import java.util.EnumSet;
@@ -176,7 +178,8 @@ public class RetriableFileCopyCommand ex
return new Path(root, ".distcp.tmp." + context.getTaskAttemptID().toString());
}
- private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream,
+ @VisibleForTesting
+ long copyBytes(FileStatus sourceFileStatus, OutputStream outStream,
int bufferSize, Mapper.Context context)
throws IOException {
Path source = sourceFileStatus.getPath();
@@ -193,6 +196,8 @@ public class RetriableFileCopyCommand ex
updateContextStatus(totalBytesRead, context, sourceFileStatus);
bytesRead = inStream.read(buf);
}
+ outStream.close();
+ outStream = null;
} finally {
IOUtils.cleanup(LOG, outStream, inStream);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java?rev=1548386&r1=1548385&r2=1548386&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java Fri Dec 6 06:57:15 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.tools;
+import static org.mockito.Mockito.*;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -36,6 +38,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.AfterClass;
+import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
@@ -282,4 +285,29 @@ public class TestCopyListing extends Sim
IOUtils.closeStream(reader);
}
}
+
+ @Test
+ public void testFailOnCloseError() throws IOException {
+ File inFile = File.createTempFile("TestCopyListingIn", null);
+ inFile.deleteOnExit();
+ File outFile = File.createTempFile("TestCopyListingOut", null);
+ outFile.deleteOnExit();
+ List<Path> srcs = new ArrayList<Path>();
+ srcs.add(new Path(inFile.toURI()));
+
+ Exception expectedEx = new IOException("boom");
+ SequenceFile.Writer writer = mock(SequenceFile.Writer.class);
+ doThrow(expectedEx).when(writer).close();
+
+ SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
+ DistCpOptions options = new DistCpOptions(srcs, new Path(outFile.toURI()));
+ Exception actualEx = null;
+ try {
+ listing.doBuildListing(writer, options);
+ } catch (Exception e) {
+ actualEx = e;
+ }
+ Assert.assertNotNull("close writer didn't fail", actualEx);
+ Assert.assertEquals(expectedEx, actualEx);
+ }
}