You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2009/05/29 01:26:48 UTC
svn commit: r779796 - in /hadoop/core/trunk: CHANGES.txt
src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java
src/tools/org/apache/hadoop/tools/DistCp.java
Author: szetszwo
Date: Thu May 28 23:26:48 2009
New Revision: 779796
URL: http://svn.apache.org/viewvc?rev=779796&view=rev
Log:
HADOOP-5620. Add an option to DistCp for preserving modification and access times. Contributed by Rodrigo Schmidt
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java
hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=779796&r1=779795&r2=779796&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu May 28 23:26:48 2009
@@ -390,7 +390,7 @@
TestNameEditsConfig, TestStartup and TestStorageRestore.
(Jakob Homan via shv)
- HADOOP-5438. Provide a single FileSystem method to create or open-for-append
+ HADOOP-5438. Provide a single FileSystem method to create or open-for-append
to a file. (He Yongqiang via dhruba)
HADOOP-5472. Change DistCp to support globbing of input paths. (Dhruba
@@ -398,6 +398,9 @@
HADOOP-5175. Don't unpack libjars on classpath. (Todd Lipcon via tomwhite)
+ HADOOP-5620. Add an option to DistCp for preserving modification and access
+ times. (Rodrigo Schmidt via szetszwo)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified: hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java?rev=779796&r1=779795&r2=779796&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java (original)
+++ hadoop/core/trunk/src/test/hdfs-with-mr/org/apache/hadoop/fs/TestCopyFiles.java Thu May 28 23:26:48 2009
@@ -380,7 +380,7 @@
deldir(hdfs, "/logs");
ToolRunner.run(new DistCp(conf), new String[] {
- "-p",
+ "-prbugp", // no t to avoid preserving mod. times
"-update",
"-log",
namenode+"/logs",
@@ -393,7 +393,7 @@
deldir(hdfs, "/logs");
ToolRunner.run(new DistCp(conf), new String[] {
- "-p",
+ "-prbugp", // no t to avoid preserving mod. times
"-overwrite",
"-log",
namenode+"/logs",
@@ -551,6 +551,32 @@
deldir(fs, "/destdat");
deldir(fs, "/srcdat");
}
+
+ {//test preserving times
+ MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
+ fs.mkdirs(new Path("/srcdat/tmpf1"));
+ fs.mkdirs(new Path("/srcdat/tmpf2"));
+ FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
+ FsPermission[] permissions = new FsPermission[srcstat.length];
+ for(int i = 0; i < srcstat.length; i++) {
+ fs.setTimes(srcstat[i].getPath(), 40, 50);
+ }
+
+ ToolRunner.run(new DistCp(conf),
+ new String[]{"-pt", nnUri+"/srcdat", nnUri+"/destdat"});
+
+ FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
+ for(int i = 0; i < dststat.length; i++) {
+ assertEquals("Modif. Time i=" + i, 40, dststat[i].getModificationTime());
+ assertEquals("Access Time i=" + i+ srcstat[i].getPath() + "-" + dststat[i].getPath(), 50, dststat[i].getAccessTime());
+ }
+
+ assertTrue("Source and destination directories do not match.",
+ checkFiles(fs, "/destdat", files));
+
+ deldir(fs, "/destdat");
+ deldir(fs, "/srcdat");
+ }
} finally {
if (cluster != null) { cluster.shutdown(); }
}
Modified: hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=779796&r1=779795&r2=779796&view=diff
==============================================================================
--- hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Thu May 28 23:26:48 2009
@@ -86,7 +86,8 @@
"\n u: user" +
"\n g: group" +
"\n p: permission" +
- "\n -p alone is equivalent to -prbugp" +
+ "\n t: modification and access times" +
+ "\n -p alone is equivalent to -prbugpt" +
"\n-i Ignore failures" +
"\n-log <logdir> Write logs to <logdir>" +
"\n-m <num_maps> Maximum number of simultaneous copies" +
@@ -146,7 +147,7 @@
}
}
static enum FileAttribute {
- BLOCK_SIZE, REPLICATION, USER, GROUP, PERMISSION;
+ BLOCK_SIZE, REPLICATION, USER, GROUP, PERMISSION, TIMES;
final char symbol;
@@ -460,7 +461,7 @@
+ ") but expected " + bytesString(srcstat.getLen())
+ " from " + srcstat.getPath());
}
- updatePermissions(srcstat, dststat);
+ updateDestStatus(srcstat, dststat);
}
// report at least once for each file
@@ -486,10 +487,10 @@
}
}
- private void updatePermissions(FileStatus src, FileStatus dst
+ private void updateDestStatus(FileStatus src, FileStatus dst
) throws IOException {
if (preserve_status) {
- DistCp.updatePermissions(src, dst, preseved, destFileSys);
+ DistCp.updateDestStatus(src, dst, preseved, destFileSys);
}
}
@@ -669,7 +670,7 @@
}
}
- private static void updatePermissions(FileStatus src, FileStatus dst,
+ private static void updateDestStatus(FileStatus src, FileStatus dst,
EnumSet<FileAttribute> preseved, FileSystem destFileSys
) throws IOException {
String owner = null;
@@ -689,6 +690,9 @@
&& !src.getPermission().equals(dst.getPermission())) {
destFileSys.setPermission(dst.getPath(), src.getPermission());
}
+ if (preseved.contains(FileAttribute.TIMES)) {
+ destFileSys.setTimes(dst.getPath(), src.getModificationTime(), src.getAccessTime());
+ }
}
static private void finalize(Configuration conf, JobConf jobconf,
@@ -713,7 +717,7 @@
FilePair pair = new FilePair();
for(; in.next(dsttext, pair); ) {
Path absdst = new Path(destPath, pair.output);
- updatePermissions(pair.input, dstfs.getFileStatus(absdst),
+ updateDestStatus(pair.input, dstfs.getFileStatus(absdst),
preseved, dstfs);
}
} finally {