You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/17 23:44:59 UTC
[12/34] git commit: HDFS-6928. 'hdfs put' command should accept
lazyPersist flag for testing. (Arpit Agarwal)
HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08cc08ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08cc08ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08cc08ad
Branch: refs/heads/branch-2
Commit: 08cc08adb3b5bba882101ec4aff1ba40c213dba4
Parents: 0aaa2b9
Author: arp <ar...@apache.org>
Authored: Thu Aug 28 15:53:12 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Fri Oct 17 13:42:01 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/FilterFileSystem.java | 13 ++++++-
.../hadoop/fs/shell/CommandWithDestination.java | 32 ++++++++++++++---
.../apache/hadoop/fs/shell/CopyCommands.java | 16 +++++----
.../java/org/apache/hadoop/fs/shell/Stat.java | 7 +++-
.../src/test/resources/testConf.xml | 28 +++++++++++----
.../src/test/resources/testHDFSConf.xml | 36 ++++++++++++++++++++
6 files changed, 113 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 52706f4..e729e67 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -179,8 +179,19 @@ public class FilterFileSystem extends FileSystem {
return fs.create(f, permission,
overwrite, bufferSize, replication, blockSize, progress);
}
-
+ @Override
+ public FSDataOutputStream create(Path f,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress,
+ ChecksumOpt checksumOpt) throws IOException {
+ return fs.create(f, permission,
+ flags, bufferSize, replication, blockSize, progress);
+ }
@Override
@Deprecated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index da67f1c..5d0d9d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
@@ -45,6 +46,9 @@ import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+
/**
* Provides: argument processing to ensure the destination is valid
* for the number of source arguments. A processPaths that accepts both
@@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand {
private boolean overwrite = false;
private boolean verifyChecksum = true;
private boolean writeChecksum = true;
+ private boolean lazyPersist = false;
/**
* The name of the raw xattr namespace. It would be nice to use
@@ -78,6 +83,10 @@ abstract class CommandWithDestination extends FsCommand {
overwrite = flag;
}
+ protected void setLazyPersist(boolean flag) {
+ lazyPersist = flag;
+ }
+
protected void setVerifyChecksum(boolean flag) {
verifyChecksum = flag;
}
@@ -379,7 +388,7 @@ abstract class CommandWithDestination extends FsCommand {
try {
PathData tempTarget = target.suffix("._COPYING_");
targetFs.setWriteChecksum(writeChecksum);
- targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.writeStreamToFile(in, tempTarget, lazyPersist);
targetFs.rename(tempTarget, target);
} finally {
targetFs.close(); // last ditch effort to ensure temp file is removed
@@ -449,10 +458,11 @@ abstract class CommandWithDestination extends FsCommand {
super(fs);
}
- void writeStreamToFile(InputStream in, PathData target) throws IOException {
+ void writeStreamToFile(InputStream in, PathData target,
+ boolean lazyPersist) throws IOException {
FSDataOutputStream out = null;
try {
- out = create(target);
+ out = create(target, lazyPersist);
IOUtils.copyBytes(in, out, getConf(), true);
} finally {
IOUtils.closeStream(out); // just in case copyBytes didn't
@@ -460,9 +470,21 @@ abstract class CommandWithDestination extends FsCommand {
}
// tag created files as temp files
- FSDataOutputStream create(PathData item) throws IOException {
+ FSDataOutputStream create(PathData item, boolean lazyPersist)
+ throws IOException {
try {
- return create(item.path, true);
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+ if (lazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+ return create(item.path,
+ null,
+ createFlags,
+ getConf().getInt("io.file.buffer.size", 4096),
+ lazyPersist ? 1 : getDefaultReplication(item.path),
+ getDefaultBlockSize(),
+ null,
+ null);
} finally { // might have been created but stream was interrupted
deleteOnExit(item.path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 3fd870c..afd1115 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -215,21 +215,25 @@ class CopyCommands {
*/
public static class Put extends CommandWithDestination {
public static final String NAME = "put";
- public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>";
+ public static final String USAGE = "[-f] [-p] [-l] <localsrc> ... <dst>";
public static final String DESCRIPTION =
"Copy files from the local file system " +
"into fs. Copying fails if the file already " +
- "exists, unless the -f flag is given. Passing " +
- "-p preserves access and modification times, " +
- "ownership and the mode. Passing -f overwrites " +
- "the destination if it already exists.\n";
+ "exists, unless the -f flag is given.\n" +
+ "Flags:\n" +
+ " -p : Preserves access and modification times, ownership and the mode.\n" +
+ " -f : Overwrites the destination if it already exists.\n" +
+ " -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
+ " replication factor of 1. This flag will result in reduced\n" +
+ " durability. Use with care.\n";
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
+ CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l");
cf.parse(args);
setOverwrite(cf.getOpt("f"));
setPreserve(cf.getOpt("p"));
+ setLazyPersist(cf.getOpt("l"));
getRemoteDestination(args);
// should have a -r option
setRecursive(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
index 652c928..8b32eb8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
* %u: User name of owner
* %y: UTC date as "yyyy-MM-dd HH:mm:ss"
* %Y: Milliseconds since January 1, 1970 UTC
+ * %l: Whether lazyPersist flag is set on the file.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -53,7 +54,8 @@ class Stat extends FsCommand {
public static final String DESCRIPTION =
"Print statistics about the file/directory at <path> " +
"in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
- "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n";
+ "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " +
+ "lazyPersist flag (%l)\n";
protected static final SimpleDateFormat timeFmt;
static {
@@ -115,6 +117,9 @@ class Stat extends FsCommand {
case 'Y':
buf.append(stat.getModificationTime());
break;
+ case 'l':
+ buf.append(stat.isLazyPersist());
+ break;
default:
// this leaves %<unknown> alone, which causes the potential for
// future format options to break strings; should use %% to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 804b504..29d4174 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -436,7 +436,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-put \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s*</expected-output>
+ <expected-output>^-put \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -444,15 +444,31 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )*</expected-output>
+ <expected-output>^\s*exists, unless the -f flag is given.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*modification times, ownership and the mode. Passing -f overwrites the( )*</expected-output>
+ <expected-output>^\s*Flags:( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*destination if it already exists.( )*</expected-output>
+ <expected-output>^\s*-p Preserves access and modification times, ownership and the mode.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*-f Overwrites the destination if it already exists.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*-l Allow DataNode to lazily persist the file to disk. Forces( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*replication factor of 1. This flag will result in reduced( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*durability. Use with care.( )*</expected-output>
</comparator>
</comparators>
</test>
@@ -467,7 +483,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-copyFromLocal \[-f\] \[-p\] <localsrc> \.\.\. <dst> :\s*</expected-output>
+ <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] <localsrc> \.\.\. <dst> :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -782,7 +798,7 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output>
+ <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
</comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08cc08ad/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 2c2a7a0..37ea234 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -4554,6 +4554,42 @@
</comparators>
</test>
+ <test> <!-- TESTED -->
+ <description>put: The LazyPersist flag is set when requested</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dirLp</command>
+ <command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+ <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dirLp/</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^true</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>put: The LazyPersist flag is not set by default</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dirLp</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
+ <command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dirLp/</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^false</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
<!-- Tests for copyFromLocal -->
<test> <!-- TESTED -->
<description>copyFromLocal: copying file into a file (absolute path)</description>