You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2017/08/02 19:28:23 UTC
[2/2] hadoop git commit: HDFS-6984. Serialize FileStatus via protobuf.
HDFS-6984. Serialize FileStatus via protobuf.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12e44e7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12e44e7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12e44e7b
Branch: refs/heads/trunk
Commit: 12e44e7bdaf53d3720a89d32f0cc2717241bd6b2
Parents: 1a1bf6b
Author: Chris Douglas <cd...@apache.org>
Authored: Wed Aug 2 12:12:48 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed Aug 2 12:12:48 2017 -0700
----------------------------------------------------------------------
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-common-project/hadoop-common/pom.xml | 1 +
.../apache/hadoop/fs/FileEncryptionInfo.java | 6 +-
.../java/org/apache/hadoop/fs/FileStatus.java | 152 ++++++++-----
.../org/apache/hadoop/fs/LocatedFileStatus.java | 64 ++++--
.../hadoop/fs/permission/FsPermission.java | 12 +
.../apache/hadoop/fs/protocolPB/PBHelper.java | 131 +++++++++++
.../hadoop/fs/protocolPB/package-info.java | 18 ++
.../apache/hadoop/io/erasurecode/ECSchema.java | 6 +-
.../hadoop-common/src/main/proto/FSProtos.proto | 69 ++++++
.../org/apache/hadoop/fs/TestFileStatus.java | 1 +
.../fs/protocolPB/TestFSSerialization.java | 85 +++++++
.../hdfs/protocol/ErasureCodingPolicy.java | 8 +-
.../hdfs/protocol/FsPermissionExtension.java | 5 +
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 228 +++++++++----------
.../hdfs/protocol/HdfsLocatedFileStatus.java | 42 ++--
.../protocol/SnapshottableDirectoryStatus.java | 10 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +++++-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 50 ++--
.../hadoop/hdfs/web/WebHdfsConstants.java | 8 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hadoop-hdfs-client/src/main/proto/acl.proto | 7 +-
.../src/main/proto/hdfs.proto | 16 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 45 ++--
.../fs/http/client/BaseTestHttpFSWith.java | 19 +-
.../dev-support/findbugsExcludeFile.xml | 12 +
.../hadoop/hdfs/protocol/SnapshotInfo.java | 2 +-
.../hadoop/hdfs/protocolPB/package-info.java | 18 ++
.../server/namenode/FSDirStatAndListingOp.java | 58 ++---
.../hdfs/server/namenode/FSDirectory.java | 20 +-
.../hdfs/server/namenode/FSNamesystem.java | 7 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +-
.../namenode/snapshot/SnapshotManager.java | 7 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 28 ++-
.../hadoop/hdfs/TestDFSClientRetries.java | 6 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 3 +-
.../hdfs/TestFileStatusSerialization.java | 153 +++++++++++++
.../java/org/apache/hadoop/hdfs/TestLease.java | 7 +-
.../hdfs/server/namenode/AclTestHelpers.java | 13 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 11 +-
41 files changed, 1054 insertions(+), 366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index de76afb..4bafd8e 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -323,6 +323,10 @@
<!-- protobuf generated code -->
<Class name="~org\.apache\.hadoop\.tracing\.TraceAdminPB.*"/>
</Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.fs\.FSProto.*"/>
+ </Match>
<!--
Manually checked, misses child thread manually syncing on parent's intrinsic lock.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 3e73bce..ad6f734 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -393,6 +393,7 @@
<include>RefreshUserMappingsProtocol.proto</include>
<include>RefreshCallQueueProtocol.proto</include>
<include>GenericRefreshProtocol.proto</include>
+ <include>FSProtos.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
index 1129e07..ce5ed56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.fs;
+import java.io.Serializable;
+
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.CipherSuite;
@@ -30,7 +32,9 @@ import static com.google.common.base.Preconditions.checkNotNull;
* an encrypted file.
*/
@InterfaceAudience.Private
-public class FileEncryptionInfo {
+public class FileEncryptionInfo implements Serializable {
+
+ private static final long serialVersionUID = 0x156abe03;
private final CipherSuite cipherSuite;
private final CryptoProtocolVersion version;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index 7ce6363..2f22ea0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -23,11 +23,15 @@ import java.io.IOException;
import java.io.InvalidObjectException;
import java.io.ObjectInputValidation;
import java.io.Serializable;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+import org.apache.hadoop.fs.FSProtos.FileStatusProto;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.fs.protocolPB.PBHelper;
import org.apache.hadoop.io.Writable;
/** Interface that represents the client side information for a file.
@@ -50,7 +54,31 @@ public class FileStatus implements Writable, Comparable<Object>,
private String owner;
private String group;
private Path symlink;
-
+ private Set<AttrFlags> attr;
+
+ private enum AttrFlags {
+ HAS_ACL,
+ HAS_CRYPT,
+ HAS_EC,
+ };
+ private static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet();
+ private static Set<AttrFlags> flags(boolean acl, boolean crypt, boolean ec) {
+ if (!(acl || crypt || ec)) {
+ return NONE;
+ }
+ EnumSet<AttrFlags> ret = EnumSet.noneOf(AttrFlags.class);
+ if (acl) {
+ ret.add(AttrFlags.HAS_ACL);
+ }
+ if (crypt) {
+ ret.add(AttrFlags.HAS_CRYPT);
+ }
+ if (ec) {
+ ret.add(AttrFlags.HAS_EC);
+ }
+ return ret;
+ }
+
public FileStatus() { this(0, false, 0, 0, 0, 0, null, null, null, null); }
//We should deprecate this soon?
@@ -79,6 +107,15 @@ public class FileStatus implements Writable, Comparable<Object>,
FsPermission permission, String owner, String group,
Path symlink,
Path path) {
+ this(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path,
+ false, false, false);
+ }
+
+ public FileStatus(long length, boolean isdir, int block_replication,
+ long blocksize, long modification_time, long access_time,
+ FsPermission permission, String owner, String group, Path symlink,
+ Path path, boolean hasAcl, boolean isEncrypted, boolean isErasureCoded) {
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
@@ -89,7 +126,7 @@ public class FileStatus implements Writable, Comparable<Object>,
this.permission = permission;
} else if (isdir) {
this.permission = FsPermission.getDirDefault();
- } else if (symlink!=null) {
+ } else if (symlink != null) {
this.permission = FsPermission.getDefault();
} else {
this.permission = FsPermission.getFileDefault();
@@ -98,6 +135,8 @@ public class FileStatus implements Writable, Comparable<Object>,
this.group = (group == null) ? "" : group;
this.symlink = symlink;
this.path = path;
+ attr = flags(hasAcl, isEncrypted, isErasureCoded);
+
// The variables isdir and symlink indicate the type:
// 1. isdir implies directory, in which case symlink must be null.
// 2. !isdir implies a file or symlink, symlink != null implies a
@@ -213,7 +252,7 @@ public class FileStatus implements Writable, Comparable<Object>,
* @return true if the underlying file or directory has ACLs set.
*/
public boolean hasAcl() {
- return permission.getAclBit();
+ return attr.contains(AttrFlags.HAS_ACL);
}
/**
@@ -222,7 +261,7 @@ public class FileStatus implements Writable, Comparable<Object>,
* @return true if the underlying file is encrypted.
*/
public boolean isEncrypted() {
- return permission.getEncryptedBit();
+ return attr.contains(AttrFlags.HAS_CRYPT);
}
/**
@@ -231,7 +270,7 @@ public class FileStatus implements Writable, Comparable<Object>,
* @return true if the underlying file or directory is erasure coded.
*/
public boolean isErasureCoded() {
- return permission.getErasureCodedBit();
+ return attr.contains(AttrFlags.HAS_EC);
}
/**
@@ -304,47 +343,6 @@ public class FileStatus implements Writable, Comparable<Object>,
public void setSymlink(final Path p) {
symlink = p;
}
-
- //////////////////////////////////////////////////
- // Writable
- //////////////////////////////////////////////////
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN);
- out.writeLong(getLen());
- out.writeBoolean(isDirectory());
- out.writeShort(getReplication());
- out.writeLong(getBlockSize());
- out.writeLong(getModificationTime());
- out.writeLong(getAccessTime());
- getPermission().write(out);
- Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN);
- Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN);
- out.writeBoolean(isSymlink());
- if (isSymlink()) {
- Text.writeString(out, getSymlink().toString(), Text.DEFAULT_MAX_LEN);
- }
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- String strPath = Text.readString(in, Text.DEFAULT_MAX_LEN);
- this.path = new Path(strPath);
- this.length = in.readLong();
- this.isdir = in.readBoolean();
- this.block_replication = in.readShort();
- blocksize = in.readLong();
- modification_time = in.readLong();
- access_time = in.readLong();
- permission.readFields(in);
- owner = Text.readString(in, Text.DEFAULT_MAX_LEN);
- group = Text.readString(in, Text.DEFAULT_MAX_LEN);
- if (in.readBoolean()) {
- this.symlink = new Path(Text.readString(in, Text.DEFAULT_MAX_LEN));
- } else {
- this.symlink = null;
- }
- }
/**
* Compare this FileStatus to another FileStatus
@@ -377,15 +375,12 @@ public class FileStatus implements Writable, Comparable<Object>,
*/
@Override
public boolean equals(Object o) {
- if (o == null) {
+ if (!(o instanceof FileStatus)) {
return false;
}
if (this == o) {
return true;
}
- if (!(o instanceof FileStatus)) {
- return false;
- }
FileStatus other = (FileStatus)o;
return this.getPath().equals(other.getPath());
}
@@ -420,7 +415,11 @@ public class FileStatus implements Writable, Comparable<Object>,
sb.append("; permission=" + permission);
sb.append("; isSymlink=" + isSymlink());
if(isSymlink()) {
- sb.append("; symlink=" + symlink);
+ try {
+ sb.append("; symlink=" + getSymlink());
+ } catch (IOException e) {
+ throw new RuntimeException("Unexpected exception", e);
+ }
}
sb.append("; hasAcl=" + hasAcl());
sb.append("; isEncrypted=" + isEncrypted());
@@ -429,6 +428,55 @@ public class FileStatus implements Writable, Comparable<Object>,
return sb.toString();
}
+ /**
+ * Read instance encoded as protobuf from stream.
+ * @param in Input stream
+ * @see PBHelper#convert(FileStatus)
+ * @deprecated Use the {@link PBHelper} and protobuf serialization directly.
+ */
+ @Override
+ @Deprecated
+ public void readFields(DataInput in) throws IOException {
+ int size = in.readInt();
+ if (size < 0) {
+ throw new IOException("Can't read FileStatusProto with negative " +
+ "size of " + size);
+ }
+ byte[] buf = new byte[size];
+ in.readFully(buf);
+ FileStatusProto proto = FileStatusProto.parseFrom(buf);
+ FileStatus other = PBHelper.convert(proto);
+ isdir = other.isDirectory();
+ length = other.getLen();
+ isdir = other.isDirectory();
+ block_replication = other.getReplication();
+ blocksize = other.getBlockSize();
+ modification_time = other.getModificationTime();
+ access_time = other.getAccessTime();
+ setPermission(other.getPermission());
+ setOwner(other.getOwner());
+ setGroup(other.getGroup());
+ setSymlink((other.isSymlink() ? other.getSymlink() : null));
+ setPath(other.getPath());
+ attr = flags(other.hasAcl(), other.isEncrypted(), other.isErasureCoded());
+ assert (isDirectory() && getSymlink() == null) || !isDirectory();
+ }
+
+ /**
+ * Write instance encoded as protobuf to stream.
+ * @param out Output stream
+ * @see PBHelper#convert(FileStatus)
+ * @deprecated Use the {@link PBHelper} and protobuf serialization directly.
+ */
+ @Override
+ @Deprecated
+ public void write(DataOutput out) throws IOException {
+ FileStatusProto proto = PBHelper.convert(this);
+ int size = proto.getSerializedSize();
+ out.writeInt(size);
+ out.write(proto.toByteArray());
+ }
+
@Override
public void validateObject() throws InvalidObjectException {
if (null == path) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
index 588fd6a..885da07 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
@@ -30,6 +30,9 @@ import org.apache.hadoop.fs.permission.FsPermission;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class LocatedFileStatus extends FileStatus {
+
+ private static final long serialVersionUID = 0x17339920;
+
private BlockLocation[] locations;
@@ -42,14 +45,18 @@ public class LocatedFileStatus extends FileStatus {
* @param stat a file status
* @param locations a file's block locations
*/
- public LocatedFileStatus(FileStatus stat, BlockLocation[] locations)
- throws IOException {
+ public LocatedFileStatus(FileStatus stat, BlockLocation[] locations) {
this(stat.getLen(), stat.isDirectory(), stat.getReplication(),
stat.getBlockSize(), stat.getModificationTime(),
- stat.getAccessTime(), stat.getPermission(), stat.getOwner(),
- stat.getGroup(), null, stat.getPath(), locations);
+ stat.getAccessTime(), stat.getPermission(),
+ stat.getOwner(), stat.getGroup(), null, stat.getPath(),
+ stat.hasAcl(), stat.isEncrypted(), stat.isErasureCoded(), locations);
if (stat.isSymlink()) {
- setSymlink(stat.getSymlink());
+ try {
+ setSymlink(stat.getSymlink());
+ } catch (IOException e) {
+ throw new RuntimeException("Unexpected exception", e);
+ }
}
}
@@ -69,24 +76,55 @@ public class LocatedFileStatus extends FileStatus {
* @param path the path's qualified name
* @param locations a file's block locations
*/
+ @Deprecated
public LocatedFileStatus(long length, boolean isdir,
int block_replication,
long blocksize, long modification_time, long access_time,
FsPermission permission, String owner, String group,
- Path symlink,
- Path path,
- BlockLocation[] locations) {
- super(length, isdir, block_replication, blocksize, modification_time,
- access_time, permission, owner, group, symlink, path);
- this.locations = locations;
+ Path symlink, Path path, BlockLocation[] locations) {
+ this(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path,
+ permission.getAclBit(), permission.getEncryptedBit(),
+ permission.getErasureCodedBit(), locations);
}
-
+
+ /**
+ * Constructor.
+ *
+ * @param length a file's length
+ * @param isdir if the path is a directory
+ * @param block_replication the file's replication factor
+ * @param blocksize a file's block size
+ * @param modification_time a file's modification time
+ * @param access_time a file's access time
+ * @param permission a file's permission
+ * @param owner a file's owner
+ * @param group a file's group
+ * @param symlink symlink if the path is a symbolic link
+ * @param path the path's qualified name
+ * @param hasAcl entity has associated ACLs
+ * @param isEncrypted entity is encrypted
+ * @param isErasureCoded entity is erasure coded
+ * @param locations a file's block locations
+ */
+ public LocatedFileStatus(long length, boolean isdir,
+ int block_replication, long blocksize, long modification_time,
+ long access_time, FsPermission permission, String owner, String group,
+ Path symlink, Path path,
+ boolean hasAcl, boolean isEncrypted, boolean isErasureCoded,
+ BlockLocation[] locations) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, symlink, path,
+ hasAcl, isEncrypted, isErasureCoded);
+ this.locations = locations;
+ }
+
/**
* Get the file's block locations
* @return the file's block locations
*/
public BlockLocation[] getBlockLocations() {
- return locations;
+ return locations;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 73ab5f6..23692de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -133,11 +133,13 @@ public class FsPermission implements Writable, Serializable,
}
@Override
+ @Deprecated
public void write(DataOutput out) throws IOException {
out.writeShort(toShort());
}
@Override
+ @Deprecated
public void readFields(DataInput in) throws IOException {
fromShort(in.readShort());
}
@@ -184,6 +186,7 @@ public class FsPermission implements Writable, Serializable,
*
* @return short extended short representation of this permission
*/
+ @Deprecated
public short toExtendedShort() {
return toShort();
}
@@ -299,7 +302,10 @@ public class FsPermission implements Writable, Serializable,
* Returns true if there is also an ACL (access control list).
*
* @return boolean true if there is also an ACL (access control list).
+ * @deprecated Get acl bit from the {@link org.apache.hadoop.fs.FileStatus}
+ * object.
*/
+ @Deprecated
public boolean getAclBit() {
// File system subclasses that support the ACL bit would override this.
return false;
@@ -307,14 +313,20 @@ public class FsPermission implements Writable, Serializable,
/**
* Returns true if the file is encrypted or directory is in an encryption zone
+ * @deprecated Get encryption bit from the
+ * {@link org.apache.hadoop.fs.FileStatus} object.
*/
+ @Deprecated
public boolean getEncryptedBit() {
return false;
}
/**
* Returns true if the file or directory is erasure coded.
+ * @deprecated Get ec bit from the {@link org.apache.hadoop.fs.FileStatus}
+ * object.
*/
+ @Deprecated
public boolean getErasureCodedBit() {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
new file mode 100644
index 0000000..aa7d8f5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.protocolPB;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.FSProtos.*;
+
+/**
+ * Utility methods aiding conversion of fs data structures.
+ */
+public final class PBHelper {
+
+ private PBHelper() {
+ // prevent construction
+ }
+
+ public static FsPermission convert(FsPermissionProto proto)
+ throws IOException {
+ return new FsPermission((short)proto.getPerm());
+ }
+
+ public static FsPermissionProto convert(FsPermission p) throws IOException {
+ FsPermissionProto.Builder bld = FsPermissionProto.newBuilder();
+ bld.setPerm(p.toShort());
+ return bld.build();
+ }
+
+ public static FileStatus convert(FileStatusProto proto) throws IOException {
+ final Path path;
+ final long length;
+ final boolean isdir;
+ final short blockReplication;
+ final long blocksize;
+ final long mtime;
+ final long atime;
+ final String owner;
+ final String group;
+ final FsPermission permission;
+ final Path symlink;
+ switch (proto.getFileType()) {
+ case FT_DIR:
+ isdir = true;
+ symlink = null;
+ blocksize = 0;
+ length = 0;
+ blockReplication = 0;
+ break;
+ case FT_SYMLINK:
+ isdir = false;
+ symlink = new Path(proto.getSymlink());
+ blocksize = 0;
+ length = 0;
+ blockReplication = 0;
+ break;
+ case FT_FILE:
+ isdir = false;
+ symlink = null;
+ blocksize = proto.getBlockSize();
+ length = proto.getLength();
+ int brep = proto.getBlockReplication();
+ if ((brep & 0xffff0000) != 0) {
+ throw new IOException(String.format("Block replication 0x%08x " +
+ "doesn't fit in 16 bits.", brep));
+ }
+ blockReplication = (short)brep;
+ break;
+ default:
+ throw new IllegalStateException("Unknown type: " + proto.getFileType());
+ }
+ path = new Path(proto.getPath());
+ mtime = proto.getModificationTime();
+ atime = proto.getAccessTime();
+ permission = convert(proto.getPermission());
+ owner = proto.getOwner();
+ group = proto.getGroup();
+ int flags = proto.getFlags();
+ return new FileStatus(length, isdir, blockReplication, blocksize,
+ mtime, atime, permission, owner, group, symlink, path,
+ (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0,
+ (flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0,
+ (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0);
+ }
+
+ public static FileStatusProto convert(FileStatus stat) throws IOException {
+ FileStatusProto.Builder bld = FileStatusProto.newBuilder();
+ bld.setPath(stat.getPath().toString());
+ if (stat.isDirectory()) {
+ bld.setFileType(FileStatusProto.FileType.FT_DIR);
+ } else if (stat.isSymlink()) {
+ bld.setFileType(FileStatusProto.FileType.FT_SYMLINK)
+ .setSymlink(stat.getSymlink().toString());
+ } else {
+ bld.setFileType(FileStatusProto.FileType.FT_FILE)
+ .setLength(stat.getLen())
+ .setBlockReplication(stat.getReplication())
+ .setBlockSize(stat.getBlockSize());
+ }
+ bld.setAccessTime(stat.getAccessTime())
+ .setModificationTime(stat.getModificationTime())
+ .setOwner(stat.getOwner())
+ .setGroup(stat.getGroup())
+ .setPermission(convert(stat.getPermission()));
+ int flags = 0;
+ flags |= stat.hasAcl() ? FileStatusProto.Flags.HAS_ACL_VALUE : 0;
+ flags |= stat.isEncrypted() ? FileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
+ flags |= stat.isErasureCoded() ? FileStatusProto.Flags.HAS_EC_VALUE : 0;
+ bld.setFlags(flags);
+ return bld.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java
new file mode 100644
index 0000000..dbc9bce
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.protocolPB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
index 4d66019..f008e24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode;
+import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -31,7 +32,10 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public final class ECSchema {
+public final class ECSchema implements Serializable {
+
+ private static final long serialVersionUID = 0x10953aa0;
+
public static final String NUM_DATA_UNITS_KEY = "numDataUnits";
public static final String NUM_PARITY_UNITS_KEY = "numParityUnits";
public static final String CODEC_NAME_KEY = "codec";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
new file mode 100644
index 0000000..6fe7980
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and stable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *stable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.fs";
+option java_outer_classname = "FSProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.fs;
+
+message FsPermissionProto {
+ required uint32 perm = 1; // UNIX-style mode bits
+}
+
+/*
+ * FileStatus encoding. Field IDs match those from HdfsFileStatusProto, but
+ * cross-serialization is not an explicitly supported use case. Unlike HDFS,
+ * most fields are optional and do not define defaults.
+ */
+message FileStatusProto {
+ enum FileType {
+ FT_DIR = 1;
+ FT_FILE = 2;
+ FT_SYMLINK = 3;
+ }
+ enum Flags {
+ HAS_ACL = 0x01; // has ACLs
+ HAS_CRYPT = 0x02; // encrypted
+ HAS_EC = 0x04; // erasure coded
+ }
+ required FileType fileType = 1;
+ required string path = 2;
+ optional uint64 length = 3;
+ optional FsPermissionProto permission = 4;
+ optional string owner = 5;
+ optional string group = 6;
+ optional uint64 modification_time = 7;
+ optional uint64 access_time = 8;
+ optional string symlink = 9;
+ optional uint32 block_replication = 10;
+ optional uint64 block_size = 11;
+ // locations = 12
+ // alias = 13
+ // childrenNum = 14
+ optional bytes encryption_data = 15;
+ // storagePolicy = 16
+ optional bytes ec_data = 17;
+ optional uint32 flags = 18 [default = 0];
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
index 1962f49..61a688e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
@@ -36,6 +36,7 @@ import org.junit.Test;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java
new file mode 100644
index 0000000..31cacf7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/protocolPB/TestFSSerialization.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.protocolPB;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import static org.apache.hadoop.fs.FSProtos.*;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Verify PB serialization of FS data structures.
+ */
+public class TestFSSerialization {
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testWritableFlagSerialization() throws Exception {
+ final Path p = new Path("hdfs://yaks:4344/dingos/f");
+ for (int i = 0; i < 0x8; ++i) {
+ final boolean acl = 0 != (i & 0x1);
+ final boolean crypt = 0 != (i & 0x2);
+ final boolean ec = 0 != (i & 0x4);
+ FileStatus stat = new FileStatus(1024L, false, 3, 1L << 31,
+ 12345678L, 87654321L, FsPermission.getFileDefault(),
+ "hadoop", "unqbbc", null, p, acl, crypt, ec);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ stat.write(dob);
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(dob.getData(), 0, dob.getLength());
+ FileStatus fstat = new FileStatus();
+ fstat.readFields(dib);
+ assertEquals(stat, fstat);
+ checkFields(stat, fstat);
+ }
+ }
+
+ @Test
+ public void testUtilitySerialization() throws Exception {
+ final Path p = new Path("hdfs://yaks:4344/dingos/f");
+ FileStatus stat = new FileStatus(1024L, false, 3, 1L << 31,
+ 12345678L, 87654321L, FsPermission.createImmutable((short)0111),
+ "hadoop", "unqbbc", null, p);
+ FileStatusProto fsp = PBHelper.convert(stat);
+ FileStatus stat2 = PBHelper.convert(fsp);
+ assertEquals(stat, stat2);
+ checkFields(stat, stat2);
+ }
+
+ private static void checkFields(FileStatus expected, FileStatus actual) {
+ assertEquals(expected.getPath(), actual.getPath());
+ assertEquals(expected.isDirectory(), actual.isDirectory());
+ assertEquals(expected.getLen(), actual.getLen());
+ assertEquals(expected.getPermission(), actual.getPermission());
+ assertEquals(expected.getOwner(), actual.getOwner());
+ assertEquals(expected.getGroup(), actual.getGroup());
+ assertEquals(expected.getModificationTime(), actual.getModificationTime());
+ assertEquals(expected.getAccessTime(), actual.getAccessTime());
+ assertEquals(expected.getReplication(), actual.getReplication());
+ assertEquals(expected.getBlockSize(), actual.getBlockSize());
+ assertEquals(expected.hasAcl(), actual.hasAcl());
+ assertEquals(expected.isEncrypted(), actual.isEncrypted());
+ assertEquals(expected.isErasureCoded(), actual.isErasureCoded());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 368a2f2..7afc377 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
+import java.io.Serializable;
+
import com.google.common.base.Preconditions;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
@@ -29,11 +31,13 @@ import org.apache.hadoop.io.erasurecode.ECSchema;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public final class ErasureCodingPolicy {
+public final class ErasureCodingPolicy implements Serializable {
+
+ private static final long serialVersionUID = 0x0079fe4e;
+ private String name;
private final ECSchema schema;
private final int cellSize;
- private String name;
private byte id;
public ErasureCodingPolicy(String name, ECSchema schema,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
index e0dd0d7..37d04e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
@@ -27,6 +27,11 @@ import org.apache.hadoop.fs.permission.FsPermission;
* done for backwards compatibility in case any existing clients assume the
* value of FsPermission is in a particular range.
*/
+
+/**
+ * @deprecated ACLs, encryption, and erasure coding are managed on FileStatus.
+ */
+@Deprecated
@InterfaceAudience.Private
public class FsPermissionExtension extends FsPermission {
private static final long serialVersionUID = 0x13c298a4;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index c386602..8438b01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hdfs.protocol;
+import java.io.IOException;
import java.net.URI;
+import java.util.EnumSet;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -31,24 +33,15 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class HdfsFileStatus {
+public class HdfsFileStatus extends FileStatus {
+
+ private static final long serialVersionUID = 0x126eb82a;
// local name of the inode that's encoded in java UTF8
- private final byte[] path;
- private final byte[] symlink; // symlink target encoded in java UTF8 or null
- private final long length;
- private final boolean isdir;
- private final short block_replication;
- private final long blocksize;
- private final long modification_time;
- private final long access_time;
- private final FsPermission permission;
- private final String owner;
- private final String group;
+ private byte[] uPath;
+ private byte[] uSymlink; // symlink target encoded in java UTF8/null
private final long fileId;
-
private final FileEncryptionInfo feInfo;
-
private final ErasureCodingPolicy ecPolicy;
// Used by dir, not including dot and dotdot. Always zero for a regular file.
@@ -58,11 +51,21 @@ public class HdfsFileStatus {
public static final byte[] EMPTY_NAME = new byte[0];
/**
+ * Set of features potentially active on an instance.
+ */
+ public enum Flags {
+ HAS_ACL,
+ HAS_CRYPT,
+ HAS_EC;
+ }
+ private final EnumSet<Flags> flags;
+
+ /**
* Constructor.
- * @param length the number of bytes the file has
- * @param isdir if the path is a directory
+ * @param length the number of bytes the file has
+ * @param isdir if the path is a directory
* @param block_replication the replication factor
- * @param blocksize the block size
+ * @param blocksize the block size
* @param modification_time modification time
* @param access_time access time
* @param permission permission
@@ -77,25 +80,18 @@ public class HdfsFileStatus {
* @param ecPolicy the erasure coding policy
*/
public HdfsFileStatus(long length, boolean isdir, int block_replication,
- long blocksize, long modification_time, long access_time,
- FsPermission permission, String owner, String group, byte[] symlink,
- byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo,
- byte storagePolicy, ErasureCodingPolicy ecPolicy) {
- this.length = length;
- this.isdir = isdir;
- this.block_replication = (short) block_replication;
- this.blocksize = blocksize;
- this.modification_time = modification_time;
- this.access_time = access_time;
- this.permission = (permission == null) ?
- ((isdir || symlink!=null) ?
- FsPermission.getDefault() :
- FsPermission.getFileDefault()) :
- permission;
- this.owner = (owner == null) ? "" : owner;
- this.group = (group == null) ? "" : group;
- this.symlink = symlink;
- this.path = path;
+ long blocksize, long modification_time,
+ long access_time, FsPermission permission,
+ EnumSet<Flags> flags, String owner, String group,
+ byte[] symlink, byte[] path, long fileId,
+ int childrenNum, FileEncryptionInfo feInfo,
+ byte storagePolicy, ErasureCodingPolicy ecPolicy) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, convert(isdir, symlink != null, permission, flags),
+ owner, group, null, null);
+ this.flags = flags;
+ this.uSymlink = symlink;
+ this.uPath = path;
this.fileId = fileId;
this.childrenNum = childrenNum;
this.feInfo = feInfo;
@@ -104,83 +100,48 @@ public class HdfsFileStatus {
}
/**
- * Get the length of this file, in bytes.
- * @return the length of this file, in bytes.
+ * Set redundant flags for compatibility with existing applications.
*/
- public final long getLen() {
- return length;
- }
-
- /**
- * Is this a directory?
- * @return true if this is a directory
- */
- public final boolean isDir() {
- return isdir;
+ protected static FsPermission convert(boolean isdir, boolean symlink,
+ FsPermission p, EnumSet<Flags> f) {
+ if (p instanceof FsPermissionExtension) {
+ // verify flags are set consistently
+ assert p.getAclBit() == f.contains(HdfsFileStatus.Flags.HAS_ACL);
+ assert p.getEncryptedBit() == f.contains(HdfsFileStatus.Flags.HAS_CRYPT);
+ assert p.getErasureCodedBit() == f.contains(HdfsFileStatus.Flags.HAS_EC);
+ return p;
+ }
+ if (null == p) {
+ if (isdir) {
+ p = FsPermission.getDirDefault();
+ } else if (symlink) {
+ p = FsPermission.getDefault();
+ } else {
+ p = FsPermission.getFileDefault();
+ }
+ }
+ return new FsPermissionExtension(p, f.contains(Flags.HAS_ACL),
+ f.contains(Flags.HAS_CRYPT), f.contains(Flags.HAS_EC));
}
- /**
- * Is this a symbolic link?
- * @return true if this is a symbolic link
- */
+ @Override
public boolean isSymlink() {
- return symlink != null;
+ return uSymlink != null;
}
- /**
- * Get the block size of the file.
- * @return the number of bytes
- */
- public final long getBlockSize() {
- return blocksize;
+ @Override
+ public boolean hasAcl() {
+ return flags.contains(Flags.HAS_ACL);
}
- /**
- * Get the replication factor of a file.
- * @return the replication factor of a file.
- */
- public final short getReplication() {
- return block_replication;
+ @Override
+ public boolean isEncrypted() {
+ return flags.contains(Flags.HAS_CRYPT);
}
- /**
- * Get the modification time of the file.
- * @return the modification time of file in milliseconds since January 1, 1970 UTC.
- */
- public final long getModificationTime() {
- return modification_time;
- }
-
- /**
- * Get the access time of the file.
- * @return the access time of file in milliseconds since January 1, 1970 UTC.
- */
- public final long getAccessTime() {
- return access_time;
- }
-
- /**
- * Get FsPermission associated with the file.
- * @return permission
- */
- public final FsPermission getPermission() {
- return permission;
- }
-
- /**
- * Get the owner of the file.
- * @return owner of the file
- */
- public final String getOwner() {
- return owner;
- }
-
- /**
- * Get the group associated with the file.
- * @return group for the file.
- */
- public final String getGroup() {
- return group;
+ @Override
+ public boolean isErasureCoded() {
+ return flags.contains(Flags.HAS_EC);
}
/**
@@ -188,7 +149,7 @@ public class HdfsFileStatus {
* @return true if the name is empty
*/
public final boolean isEmptyLocalName() {
- return path.length == 0;
+ return uPath.length == 0;
}
/**
@@ -196,7 +157,7 @@ public class HdfsFileStatus {
* @return the local name in string
*/
public final String getLocalName() {
- return DFSUtilClient.bytes2String(path);
+ return DFSUtilClient.bytes2String(uPath);
}
/**
@@ -204,7 +165,7 @@ public class HdfsFileStatus {
* @return the local name in java UTF8
*/
public final byte[] getLocalNameInBytes() {
- return path;
+ return uPath;
}
/**
@@ -238,16 +199,24 @@ public class HdfsFileStatus {
return new Path(parent, getLocalName());
}
- /**
- * Get the string representation of the symlink.
- * @return the symlink as a string.
- */
- public final String getSymlink() {
- return DFSUtilClient.bytes2String(symlink);
+ @Override
+ public Path getSymlink() throws IOException {
+ if (isSymlink()) {
+ return new Path(DFSUtilClient.bytes2String(uSymlink));
+ }
+ throw new IOException("Path " + getPath() + " is not a symbolic link");
+ }
+
+ @Override
+ public void setSymlink(Path sym) {
+ uSymlink = DFSUtilClient.string2Bytes(sym.toString());
}
+ /**
+ * Opaque referant for the symlink, to be resolved at the client.
+ */
public final byte[] getSymlinkInBytes() {
- return symlink;
+ return uSymlink;
}
public final long getFileId() {
@@ -275,13 +244,30 @@ public class HdfsFileStatus {
return storagePolicy;
}
- public final FileStatus makeQualified(URI defaultUri, Path path) {
- return new FileStatus(getLen(), isDir(), getReplication(),
- getBlockSize(), getModificationTime(),
- getAccessTime(),
- getPermission(), getOwner(), getGroup(),
- isSymlink() ? new Path(getSymlink()) : null,
- (getFullPath(path)).makeQualified(
- defaultUri, null)); // fully-qualify path
+ @Override
+ public boolean equals(Object o) {
+ // satisfy findbugs
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // satisfy findbugs
+ return super.hashCode();
+ }
+
+ /**
+ * Resolve the short name of the Path given the URI, parent provided. This
+ * FileStatus reference will not contain a valid Path until it is resolved
+ * by this method.
+ * @param defaultUri FileSystem to fully qualify HDFS path.
+ * @param parent Parent path of this element.
+ * @return Reference to this instance.
+ */
+ public final FileStatus makeQualified(URI defaultUri, Path parent) {
+ // fully-qualify path
+ setPath(getFullPath(parent).makeQualified(defaultUri, null));
+ return this; // API compatibility
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
index 0fd2039..b82a860 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.protocol;
import java.net.URI;
+import java.util.EnumSet;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -34,7 +35,14 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class HdfsLocatedFileStatus extends HdfsFileStatus {
- private final LocatedBlocks locations;
+
+ private static final long serialVersionUID = 0x23c73328;
+
+ /**
+ * Left transient, because {@link #makeQualifiedLocated(URI,Path)}
+ * is the user-facing type.
+ */
+ private transient LocatedBlocks locations;
/**
* Constructor
@@ -56,12 +64,12 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
*/
public HdfsLocatedFileStatus(long length, boolean isdir,
int block_replication, long blocksize, long modification_time,
- long access_time, FsPermission permission, String owner, String group,
- byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
- int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
- ErasureCodingPolicy ecPolicy) {
+ long access_time, FsPermission permission, EnumSet<Flags> flags,
+ String owner, String group, byte[] symlink, byte[] path, long fileId,
+ LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo,
+ byte storagePolicy, ErasureCodingPolicy ecPolicy) {
super(length, isdir, block_replication, blocksize, modification_time,
- access_time, permission, owner, group, symlink, path, fileId,
+ access_time, permission, flags, owner, group, symlink, path, fileId,
childrenNum, feInfo, storagePolicy, ecPolicy);
this.locations = locations;
}
@@ -72,13 +80,21 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
Path path) {
- return new LocatedFileStatus(getLen(), isDir(), getReplication(),
- getBlockSize(), getModificationTime(),
- getAccessTime(),
- getPermission(), getOwner(), getGroup(),
- isSymlink() ? new Path(getSymlink()) : null,
- (getFullPath(path)).makeQualified(
- defaultUri, null), // fully-qualify path
+ makeQualified(defaultUri, path);
+ return new LocatedFileStatus(this,
DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
}
+
+ @Override
+ public boolean equals(Object o) {
+ // satisfy findbugs
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // satisfy findbugs
+ return super.hashCode();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
index 583d027..61e5316 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
@@ -21,6 +21,7 @@ import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.Comparator;
import java.util.Date;
+import java.util.EnumSet;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -57,11 +58,12 @@ public class SnapshottableDirectoryStatus {
private final byte[] parentFullPath;
public SnapshottableDirectoryStatus(long modification_time, long access_time,
- FsPermission permission, String owner, String group, byte[] localName,
- long inodeId, int childrenNum,
- int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
+ FsPermission permission, EnumSet<HdfsFileStatus.Flags> flags,
+ String owner, String group, byte[] localName, long inodeId,
+ int childrenNum, int snapshotNumber, int snapshotQuota,
+ byte[] parentFullPath) {
this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
- access_time, permission, owner, group, null, localName, inodeId,
+ access_time, permission, flags, owner, group, null, localName, inodeId,
childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
null);
this.snapshotNumber = snapshotNumber;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index feb3061..a0d2c5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntrySco
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockFlagProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
@@ -149,7 +150,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.Sto
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
@@ -1142,7 +1142,7 @@ public class PBHelperClient {
}
public static FsPermission convert(FsPermissionProto p) {
- return new FsPermissionExtension((short)p.getPerm());
+ return new FsPermission((short)p.getPerm());
}
private static Event.CreateEvent.INodeType createTypeConvert(
@@ -1501,10 +1501,14 @@ public class PBHelperClient {
return null;
}
final HdfsFileStatusProto status = sdirStatusProto.getDirStatus();
+ EnumSet<HdfsFileStatus.Flags> flags = status.hasFlags()
+ ? convertFlags(status.getFlags())
+ : convertFlags(status.getPermission());
return new SnapshottableDirectoryStatus(
status.getModificationTime(),
status.getAccessTime(),
convert(status.getPermission()),
+ flags,
status.getOwner(),
status.getGroup(),
status.getPath().toByteArray(),
@@ -1546,17 +1550,23 @@ public class PBHelperClient {
}
public static FsPermissionProto convert(FsPermission p) {
- return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build();
+ return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
}
public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
- if (fs == null)
+ if (fs == null) {
return null;
+ }
+ EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags()
+ ? convertFlags(fs.getFlags())
+ : convertFlags(fs.getPermission());
return new HdfsLocatedFileStatus(
fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
fs.getBlockReplication(), fs.getBlocksize(),
fs.getModificationTime(), fs.getAccessTime(),
- convert(fs.getPermission()), fs.getOwner(), fs.getGroup(),
+ convert(fs.getPermission()),
+ flags,
+ fs.getOwner(), fs.getGroup(),
fs.getFileType().equals(FileType.IS_SYMLINK) ?
fs.getSymlink().toByteArray() : null,
fs.getPath().toByteArray(),
@@ -1569,6 +1579,47 @@ public class PBHelperClient {
fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null);
}
+ private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) {
+ EnumSet<HdfsFileStatus.Flags> f =
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
+ for (HdfsFileStatusProto.Flags pbf : HdfsFileStatusProto.Flags.values()) {
+ if ((pbf.getNumber() & flags) != 0) {
+ switch (pbf) {
+ case HAS_ACL:
+ f.add(HdfsFileStatus.Flags.HAS_ACL);
+ break;
+ case HAS_CRYPT:
+ f.add(HdfsFileStatus.Flags.HAS_CRYPT);
+ break;
+ case HAS_EC:
+ f.add(HdfsFileStatus.Flags.HAS_EC);
+ break;
+ default:
+ // ignore unknown
+ break;
+ }
+ }
+ }
+ return f;
+ }
+
+ private static EnumSet<HdfsFileStatus.Flags> convertFlags(
+ FsPermissionProto pbp) {
+ EnumSet<HdfsFileStatus.Flags> f =
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
+ FsPermission p = new FsPermissionExtension((short)pbp.getPerm());
+ if (p.getAclBit()) {
+ f.add(HdfsFileStatus.Flags.HAS_ACL);
+ }
+ if (p.getEncryptedBit()) {
+ f.add(HdfsFileStatus.Flags.HAS_CRYPT);
+ }
+ if (p.getErasureCodedBit()) {
+ f.add(HdfsFileStatus.Flags.HAS_EC);
+ }
+ return f;
+ }
+
public static CorruptFileBlocks convert(CorruptFileBlocksProto c) {
if (c == null)
return null;
@@ -2082,6 +2133,10 @@ public class PBHelperClient {
builder.setEcPolicy(convertErasureCodingPolicy(
fs.getErasureCodingPolicy()));
}
+ int flags = fs.hasAcl() ? HdfsFileStatusProto.Flags.HAS_ACL_VALUE : 0;
+ flags |= fs.isEncrypted() ? HdfsFileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
+ flags |= fs.isErasureCoded() ? HdfsFileStatusProto.Flags.HAS_EC_VALUE : 0;
+ builder.setFlags(flags);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 5e9396e..7ec5fe5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -61,6 +60,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -97,17 +97,8 @@ class JsonUtilClient {
}
/** Convert a string to a FsPermission object. */
- static FsPermission toFsPermission(
- final String s, Boolean aclBit, Boolean encBit, Boolean erasureBit) {
- FsPermission perm = new FsPermission(Short.parseShort(s, 8));
- final boolean aBit = (aclBit != null) ? aclBit : false;
- final boolean eBit = (encBit != null) ? encBit : false;
- final boolean ecBit = (erasureBit != null) ? erasureBit : false;
- if (aBit || eBit || ecBit) {
- return new FsPermissionExtension(perm, aBit, eBit, ecBit);
- } else {
- return perm;
- }
+ static FsPermission toFsPermission(final String s) {
+ return null == s ? null : new FsPermission(Short.parseShort(s, 8));
}
/** Convert a Json map to a HdfsFileStatus object. */
@@ -128,10 +119,23 @@ class JsonUtilClient {
final long len = ((Number) m.get("length")).longValue();
final String owner = (String) m.get("owner");
final String group = (String) m.get("group");
- final FsPermission permission = toFsPermission((String) m.get("permission"),
- (Boolean) m.get("aclBit"),
- (Boolean) m.get("encBit"),
- (Boolean) m.get("ecBit"));
+ final FsPermission permission = toFsPermission((String)m.get("permission"));
+
+ Boolean aclBit = (Boolean) m.get("aclBit");
+ Boolean encBit = (Boolean) m.get("encBit");
+ Boolean erasureBit = (Boolean) m.get("ecBit");
+ EnumSet<HdfsFileStatus.Flags> f =
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
+ if (aclBit != null && aclBit) {
+ f.add(HdfsFileStatus.Flags.HAS_ACL);
+ }
+ if (encBit != null && encBit) {
+ f.add(HdfsFileStatus.Flags.HAS_CRYPT);
+ }
+ if (erasureBit != null && erasureBit) {
+ f.add(HdfsFileStatus.Flags.HAS_EC);
+ }
+
final long aTime = ((Number) m.get("accessTime")).longValue();
final long mTime = ((Number) m.get("modificationTime")).longValue();
final long blockSize = ((Number) m.get("blockSize")).longValue();
@@ -143,11 +147,11 @@ class JsonUtilClient {
final byte storagePolicy = m.containsKey("storagePolicy") ?
(byte) ((Number) m.get("storagePolicy")).longValue() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- return new HdfsFileStatus(len, type == WebHdfsConstants.PathType.DIRECTORY,
- replication, blockSize, mTime, aTime, permission, owner, group,
- symlink, DFSUtilClient.string2Bytes(localName),
- fileId, childrenNum, null,
- storagePolicy, null);
+ return new HdfsFileStatus(len,
+ type == WebHdfsConstants.PathType.DIRECTORY, replication, blockSize,
+ mTime, aTime, permission, f, owner, group, symlink,
+ DFSUtilClient.string2Bytes(localName), fileId, childrenNum,
+ null, storagePolicy, null);
}
static HdfsFileStatus[] toHdfsFileStatusArray(final Map<?, ?> json) {
@@ -465,9 +469,7 @@ class JsonUtilClient {
aclStatusBuilder.stickyBit((Boolean) m.get("stickyBit"));
String permString = (String) m.get("permission");
if (permString != null) {
- final FsPermission permission = toFsPermission(permString,
- (Boolean) m.get("aclBit"), (Boolean) m.get("encBit"),
- (Boolean) m.get("ecBit"));
+ final FsPermission permission = toFsPermission(permString);
aclStatusBuilder.setPermission(permission);
}
final List<?> entries = (List<?>) m.get("entries");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
index 50da899..f690dd0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsConstants.java
@@ -32,7 +32,13 @@ public class WebHdfsConstants {
FILE, DIRECTORY, SYMLINK;
static PathType valueOf(HdfsFileStatus status) {
- return status.isDir()? DIRECTORY: status.isSymlink()? SYMLINK: FILE;
+ if (status.isDirectory()) {
+ return DIRECTORY;
+ }
+ if (status.isSymlink()) {
+ return SYMLINK;
+ }
+ return FILE;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 3861cba..1159e50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1016,15 +1016,7 @@ public class WebHdfsFileSystem extends FileSystem
public FileStatus getFileStatus(Path f) throws IOException {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_FILE_STATUS);
- return makeQualified(getHdfsFileStatus(f), f);
- }
-
- private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
- return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
- f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
- f.getPermission(), f.getOwner(), f.getGroup(),
- f.isSymlink() ? new Path(f.getSymlink()) : null,
- f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
+ return getHdfsFileStatus(f).makeQualified(getUri(), f);
}
@Override
@@ -1507,6 +1499,7 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
+ final URI fsUri = getUri();
final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
return new FsPathResponseRunner<FileStatus[]>(op, f) {
@Override
@@ -1515,7 +1508,7 @@ public class WebHdfsFileSystem extends FileSystem
JsonUtilClient.toHdfsFileStatusArray(json);
final FileStatus[] statuses = new FileStatus[hdfsStatuses.length];
for (int i = 0; i < hdfsStatuses.length; i++) {
- statuses[i] = makeQualified(hdfsStatuses[i], f);
+ statuses[i] = hdfsStatuses[i].makeQualified(fsUri, f);
}
return statuses;
@@ -1541,10 +1534,11 @@ public class WebHdfsFileSystem extends FileSystem
}
}.run();
// Qualify the returned FileStatus array
+ final URI fsUri = getUri();
final HdfsFileStatus[] statuses = listing.getPartialListing();
FileStatus[] qualified = new FileStatus[statuses.length];
for (int i = 0; i < statuses.length; i++) {
- qualified[i] = makeQualified(statuses[i], f);
+ qualified[i] = statuses[i].makeQualified(fsUri, f);
}
return new DirectoryEntries(qualified, listing.getLastName(),
listing.hasMore());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto
index bb7fdb0..c2529c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/acl.proto
@@ -21,7 +21,12 @@ option java_outer_classname = "AclProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;
-import "hdfs.proto";
+/**
+ * File or Directory permision - same spec as posix
+ */
+message FsPermissionProto {
+ required uint32 perm = 1; // Actually a short - only 16bits used
+}
message AclEntryProto {
enum AclEntryScopeProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index b306fcf..7109980 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -32,6 +32,7 @@ option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "Security.proto";
+import "acl.proto";
/**
* Extended block idenfies a block
@@ -197,13 +198,6 @@ message CorruptFileBlocksProto {
}
/**
- * File or Directory permision - same spec as posix
- */
-message FsPermissionProto {
- required uint32 perm = 1; // Actually a short - only 16bits used
-}
-
-/**
* Types of recognized storage media.
*/
enum StorageTypeProto {
@@ -388,6 +382,11 @@ message HdfsFileStatusProto {
IS_FILE = 2;
IS_SYMLINK = 3;
}
+ enum Flags {
+ HAS_ACL = 0x01; // has ACLs
+ HAS_CRYPT = 0x02; // encrypted
+ HAS_EC = 0x04; // erasure coded
+ }
required FileType fileType = 1;
required bytes path = 2; // local name of inode encoded java UTF8
required uint64 length = 3;
@@ -415,6 +414,9 @@ message HdfsFileStatusProto {
// Optional field for erasure coding
optional ErasureCodingPolicyProto ecPolicy = 17;
+
+ // Set of flags
+ optional uint32 flags = 18 [default = 0];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 1ab890f..d139100 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -1047,18 +1047,7 @@ public class HttpFSFileSystem extends FileSystem
/** Convert a string to a FsPermission object. */
static FsPermission toFsPermission(JSONObject json) {
final String s = (String) json.get(PERMISSION_JSON);
- final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
- final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
- final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON);
- FsPermission perm = new FsPermission(Short.parseShort(s, 8));
- final boolean aBit = (aclBit != null) ? aclBit : false;
- final boolean eBit = (encBit != null) ? encBit : false;
- final boolean ecBit = (erasureBit != null) ? erasureBit : false;
- if (aBit || eBit || ecBit) {
- return new FsPermissionExtension(perm, aBit, eBit, ecBit);
- } else {
- return perm;
- }
+ return new FsPermission(Short.parseShort(s, 8));
}
private FileStatus createFileStatus(Path parent, JSONObject json) {
@@ -1073,23 +1062,23 @@ public class HttpFSFileSystem extends FileSystem
long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
long blockSize = (Long) json.get(BLOCK_SIZE_JSON);
short replication = ((Long) json.get(REPLICATION_JSON)).shortValue();
- FileStatus fileStatus = null;
-
- switch (type) {
- case FILE:
- case DIRECTORY:
- fileStatus = new FileStatus(len, (type == FILE_TYPE.DIRECTORY),
- replication, blockSize, mTime, aTime,
- permission, owner, group, path);
- break;
- case SYMLINK:
- Path symLink = null;
- fileStatus = new FileStatus(len, false,
- replication, blockSize, mTime, aTime,
- permission, owner, group, symLink,
- path);
+
+ final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
+ final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
+ final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON);
+ final boolean aBit = (aclBit != null) ? aclBit : false;
+ final boolean eBit = (encBit != null) ? encBit : false;
+ final boolean ecBit = (erasureBit != null) ? erasureBit : false;
+ if (aBit || eBit || ecBit) {
+ // include this for compatibility with 2.x
+ FsPermissionExtension deprecatedPerm =
+ new FsPermissionExtension(permission, aBit, eBit, ecBit);
+ return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ replication, blockSize, mTime, aTime, deprecatedPerm, owner, group,
+ null, path, aBit, eBit, ecBit);
}
- return fileStatus;
+ return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ replication, blockSize, mTime, aTime, permission, owner, group, path);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index e23093e..ca11c66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -852,10 +852,11 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
}
}
- private static void assertSameAclBit(FileSystem expected, FileSystem actual,
+ private static void assertSameAcls(FileSystem expected, FileSystem actual,
Path path) throws IOException {
FileStatus expectedFileStatus = expected.getFileStatus(path);
FileStatus actualFileStatus = actual.getFileStatus(path);
+ assertEquals(actualFileStatus.hasAcl(), expectedFileStatus.hasAcl());
assertEquals(actualFileStatus.getPermission().getAclBit(),
expectedFileStatus.getPermission().getAclBit());
}
@@ -888,31 +889,31 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
AclStatus proxyAclStat = proxyFs.getAclStatus(path);
AclStatus httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, path);
+ assertSameAcls(httpfs, proxyFs, path);
httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet,true));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, path);
+ assertSameAcls(httpfs, proxyFs, path);
httpfs.modifyAclEntries(path, AclEntry.parseAclSpec(aclUser2, true));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, path);
+ assertSameAcls(httpfs, proxyFs, path);
httpfs.removeAclEntries(path, AclEntry.parseAclSpec(rmAclUser1, false));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, path);
+ assertSameAcls(httpfs, proxyFs, path);
httpfs.removeAcl(path);
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, path);
+ assertSameAcls(httpfs, proxyFs, path);
}
/**
@@ -935,21 +936,21 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
AclStatus proxyAclStat = proxyFs.getAclStatus(dir);
AclStatus httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, dir);
+ assertSameAcls(httpfs, proxyFs, dir);
/* Set a default ACL on the directory */
httpfs.setAcl(dir, (AclEntry.parseAclSpec(defUser1,true)));
proxyAclStat = proxyFs.getAclStatus(dir);
httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, dir);
+ assertSameAcls(httpfs, proxyFs, dir);
/* Remove the default ACL */
httpfs.removeDefaultAcl(dir);
proxyAclStat = proxyFs.getAclStatus(dir);
httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
- assertSameAclBit(httpfs, proxyFs, dir);
+ assertSameAcls(httpfs, proxyFs, dir);
}
private void testEncryption() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index be54efb..2a7824a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -252,4 +252,16 @@
<Class name="org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture" />
<Bug pattern="NS_DANGEROUS_NON_SHORT_CIRCUIT" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil$1" />
+ <Method name="visitFile" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+ </Match>
+ <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not.
+ Defensible compatibility choices over time create odd corners. -->
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" />
+ <Field name="locations" />
+ <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
index 3ddfc85..676e827 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e44e7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
new file mode 100644
index 0000000..6233024
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org