You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tg...@apache.org on 2012/08/01 16:03:01 UTC
svn commit: r1368003 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/shell/
src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves
Date: Wed Aug 1 14:03:01 2012
New Revision: 1368003
URL: http://svn.apache.org/viewvc?rev=1368003&view=rev
Log:
merge -r 1368001:1368002 from trunk. FIXES: HADOOP-8633
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
- copied unchanged from r1368002, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368003&r1=1368002&r2=1368003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug 1 14:03:01 2012
@@ -692,6 +692,9 @@ Release 0.23.3 - UNRELEASED
org.apache.hadoop.classification.InterfaceAudience not found (Trevor
Robinson via tgraves)
+ HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp
+ via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368003&r1=1368002&r2=1368003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Aug 1 14:03:01 2012
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
@@ -232,31 +234,65 @@ abstract class CommandWithDestination ex
if (target.exists && (target.stat.isDirectory() || !overwrite)) {
throw new PathExistsException(target.toString());
}
- target.fs.setWriteChecksum(writeChecksum);
- PathData tempFile = null;
+ TargetFileSystem targetFs = new TargetFileSystem(target.fs);
try {
- tempFile = target.createTempFile(target+"._COPYING_");
- FSDataOutputStream out = target.fs.create(tempFile.path, true);
- IOUtils.copyBytes(in, out, getConf(), true);
+ PathData tempTarget = target.suffix("._COPYING_");
+ targetFs.setWriteChecksum(writeChecksum);
+ targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.rename(tempTarget, target);
+ } finally {
+ targetFs.close(); // last ditch effort to ensure temp file is removed
+ }
+ }
+
+ // Helper filter filesystem that registers created files as temp files to
+ // be deleted on exit unless successfully renamed
+ private static class TargetFileSystem extends FilterFileSystem {
+ TargetFileSystem(FileSystem fs) {
+ super(fs);
+ }
+
+ void writeStreamToFile(InputStream in, PathData target) throws IOException {
+ FSDataOutputStream out = null;
+ try {
+ out = create(target);
+ IOUtils.copyBytes(in, out, getConf(), true);
+ } finally {
+ IOUtils.closeStream(out); // just in case copyBytes didn't
+ }
+ }
+
+ // tag created files as temp files
+ FSDataOutputStream create(PathData item) throws IOException {
+ try {
+ return create(item.path, true);
+ } finally { // might have been created but stream was interrupted
+ deleteOnExit(item.path);
+ }
+ }
+
+ void rename(PathData src, PathData target) throws IOException {
// the rename method with an option to delete the target is deprecated
- if (target.exists && !target.fs.delete(target.path, false)) {
+ if (target.exists && !delete(target.path, false)) {
// too bad we don't know why it failed
PathIOException e = new PathIOException(target.toString());
e.setOperation("delete");
throw e;
}
- if (!tempFile.fs.rename(tempFile.path, target.path)) {
+ if (!rename(src.path, target.path)) {
// too bad we don't know why it failed
- PathIOException e = new PathIOException(tempFile.toString());
+ PathIOException e = new PathIOException(src.toString());
e.setOperation("rename");
e.setTargetPath(target.toString());
throw e;
}
- tempFile = null;
- } finally {
- if (tempFile != null) {
- tempFile.fs.delete(tempFile.path, false);
- }
+ // cancel delete on exit if rename is successful
+ cancelDeleteOnExit(src.path);
+ }
+ @Override
+ public void close() {
+ // purge any remaining temp files, but don't close underlying fs
+ processDeleteOnExit();
}
}
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1368003&r1=1368002&r2=1368003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Wed Aug 1 14:03:01 2012
@@ -60,7 +60,7 @@ public class PathData implements Compara
* @throws IOException if anything goes wrong...
*/
public PathData(String pathString, Configuration conf) throws IOException {
- this(FileSystem.get(URI.create(pathString), conf), pathString);
+ this(FileSystem.get(stringToUri(pathString), conf), pathString);
}
/**
@@ -170,16 +170,13 @@ public class PathData implements Compara
}
/**
- * Returns a temporary file for this PathData with the given extension.
- * The file will be deleted on exit.
- * @param extension for the temporary file
+ * Returns a new PathData with the given extension.
+ * @param extension for the suffix
* @return PathData
* @throws IOException shouldn't happen
*/
- public PathData createTempFile(String extension) throws IOException {
- PathData tmpFile = new PathData(fs, uri+"._COPYING_");
- fs.deleteOnExit(tmpFile.path);
- return tmpFile;
+ public PathData suffix(String extension) throws IOException {
+ return new PathData(fs, this+extension);
}
/**