You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2014/08/08 19:58:02 UTC
svn commit: r1616840 - in
/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common:
./ src/main/java/org/apache/hadoop/fs/shell/ src/site/apt/
Author: clamb
Date: Fri Aug 8 17:58:01 2014
New Revision: 1616840
URL: http://svn.apache.org/r1616840
Log:
HADOOP-10919. Copy command should preserve raw.* namespace extended attributes. (clamb)
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt?rev=1616840&r1=1616839&r2=1616840&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt Fri Aug 8 17:58:01 2014
@@ -46,6 +46,9 @@ fs-encryption (Unreleased)
HADOOP-10853. Refactor get instance of CryptoCodec and support create via
algorithm/mode/padding. (Yi Liu)
+ HADOOP-10919. Copy command should preserve raw.* namespace
+ extended attributes. (clamb)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1616840&r1=1616839&r2=1616840&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Fri Aug 8 17:58:01 2014
@@ -58,6 +58,17 @@ abstract class CommandWithDestination ex
private boolean writeChecksum = true;
/**
+ * The name of the raw xattr namespace. It would be nice to use
+ * XAttr.RAW.name() but we can't reference the hadoop-hdfs project.
+ */
+ private final String RAW = "raw.";
+
+ /**
+ * The name of the reserved raw directory.
+ */
+ private final String RESERVED_RAW = "/.reserved/raw";
+
+ /**
*
* This method is used to enable the force(-f) option while copying the files.
*
@@ -231,7 +242,7 @@ abstract class CommandWithDestination ex
/**
* Called with a source and target destination pair
* @param src for the operation
- * @param target for the operation
+ * @param dst for the operation
* @throws IOException if anything goes wrong
*/
protected void processPath(PathData src, PathData dst) throws IOException {
@@ -253,6 +264,8 @@ abstract class CommandWithDestination ex
// modify dst as we descend to append the basename of the
// current directory being processed
dst = getTargetPath(src);
+ final boolean preserveRawXattrs =
+ checkPathsForReservedRaw(src.path, dst.path);
if (dst.exists) {
if (!dst.stat.isDirectory()) {
throw new PathIsNotDirectoryException(dst.toString());
@@ -268,7 +281,7 @@ abstract class CommandWithDestination ex
}
super.recursePath(src);
if (dst.stat.isDirectory()) {
- preserveAttributes(src, dst);
+ preserveAttributes(src, dst, preserveRawXattrs);
}
} finally {
dst = savedDst;
@@ -295,19 +308,61 @@ abstract class CommandWithDestination ex
* @param target where to copy the item
* @throws IOException if copy fails
*/
- protected void copyFileToTarget(PathData src, PathData target) throws IOException {
+ protected void copyFileToTarget(PathData src, PathData target)
+ throws IOException {
+ final boolean preserveRawXattrs =
+ checkPathsForReservedRaw(src.path, target.path);
src.fs.setVerifyChecksum(verifyChecksum);
InputStream in = null;
try {
in = src.fs.open(src.path);
copyStreamToTarget(in, target);
- preserveAttributes(src, target);
+ preserveAttributes(src, target, preserveRawXattrs);
} finally {
IOUtils.closeStream(in);
}
}
/**
+ * Check the source and target paths to ensure that they are either both in
+ * /.reserved/raw or neither in /.reserved/raw. If neither src nor target are
+ * in /.reserved/raw, then return false, indicating not to preserve raw.*
+ * xattrs. If both src/target are in /.reserved/raw, then return true,
+ * indicating raw.* xattrs should be preserved. If only one of src/target is
+ * in /.reserved/raw then throw an exception.
+ *
+ * @param src The source path to check. This should be a fully-qualified
+ * path, not relative.
+ * @param target The target path to check. This should be a fully-qualified
+ * path, not relative.
+ * @return true if raw.* xattrs should be preserved.
+ * @throws PathOperationException is only one of src/target are in
+ * /.reserved/raw.
+ */
+ private boolean checkPathsForReservedRaw(Path src, Path target)
+ throws PathOperationException {
+ final boolean srcIsRR = Path.getPathWithoutSchemeAndAuthority(src).
+ toString().startsWith(RESERVED_RAW);
+ final boolean dstIsRR = Path.getPathWithoutSchemeAndAuthority(target).
+ toString().startsWith(RESERVED_RAW);
+ boolean preserveRawXattrs = false;
+ if (srcIsRR && !dstIsRR) {
+ final String s = "' copy from '" + RESERVED_RAW + "' to non '" +
+ RESERVED_RAW + "'. Either both source and target must be in '" +
+ RESERVED_RAW + "' or neither.";
+ throw new PathOperationException("'" + src.toString() + s);
+ } else if (!srcIsRR && dstIsRR) {
+ final String s = "' copy from non '" + RESERVED_RAW +"' to '" +
+ RESERVED_RAW + "'. Either both source and target must be in '" +
+ RESERVED_RAW + "' or neither.";
+ throw new PathOperationException("'" + dst.toString() + s);
+ } else if (srcIsRR && dstIsRR) {
+ preserveRawXattrs = true;
+ }
+ return preserveRawXattrs;
+ }
+
+ /**
* Copies the stream contents to a temporary file. If the copy is
* successful, the temporary file will be renamed to the real path,
* else the temporary file will be deleted.
@@ -337,9 +392,11 @@ abstract class CommandWithDestination ex
* attribute to preserve.
* @param src source to preserve
* @param target where to preserve attributes
+ * @param preserveRawXAttrs true if raw.* xattrs should be preserved
* @throws IOException if fails to preserve attributes
*/
- protected void preserveAttributes(PathData src, PathData target)
+ protected void preserveAttributes(PathData src, PathData target,
+ boolean preserveRawXAttrs)
throws IOException {
if (shouldPreserve(FileAttribute.TIMESTAMPS)) {
target.fs.setTimes(
@@ -369,13 +426,17 @@ abstract class CommandWithDestination ex
target.fs.setAcl(target.path, srcFullEntries);
}
}
- if (shouldPreserve(FileAttribute.XATTR)) {
+ final boolean preserveXAttrs = shouldPreserve(FileAttribute.XATTR);
+ if (preserveXAttrs || preserveRawXAttrs) {
Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
if (srcXAttrs != null) {
Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
while (iter.hasNext()) {
Entry<String, byte[]> entry = iter.next();
- target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
+ final String xattrName = entry.getKey();
+ if (xattrName.startsWith(RAW) || preserveXAttrs) {
+ target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
+ }
}
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1616840&r1=1616839&r2=1616840&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Fri Aug 8 17:58:01 2014
@@ -143,7 +143,11 @@ class CopyCommands {
"timestamps, ownership, permission. If -pa is specified, " +
"then preserves permission also because ACL is a super-set of " +
"permission. Passing -f overwrites the destination if it " +
- "already exists.\n";
+ "already exists. raw namespace extended attributes are preserved " +
+ "if (1) they are supported (HDFS only) and, (2) all of the source and " +
+ "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
+ "xattr preservation is determined solely by the presence (or absence) " +
+ "of the /.reserved/raw prefix and not by the -p option.\n";
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1616840&r1=1616839&r2=1616840&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Fri Aug 8 17:58:01 2014
@@ -164,15 +164,22 @@ cp
Copy files from source to destination. This command allows multiple sources
as well in which case the destination must be a directory.
+ 'raw.*' namespace extended attributes are preserved if (1) the source and
+ destination filesystems support them (HDFS only), and (2) all source and
+ destination pathnames are in the /.reserved/raw hierarchy. Determination of
+ whether raw.* namespace xattrs are preserved is independent of the
+ -p (preserve) flag.
+
Options:
* The -f option will overwrite the destination if it already exists.
- * The -p option will preserve file attributes [topx] (timestamps,
+ * The -p option will preserve file attributes [topx] (timestamps,
ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
then preserves timestamps, ownership, permission. If -pa is specified,
then preserves permission also because ACL is a super-set of
- permission.
+ permission. Determination of whether raw namespace extended attributes
+ are preserved is independent of the -p flag.
Example: