You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2017/02/01 18:19:57 UTC
hadoop git commit: HADOOP-13895. Make FileStatus Serializable
Repository: hadoop
Updated Branches:
refs/heads/trunk 3619ae32b -> 59c5f1878
HADOOP-13895. Make FileStatus Serializable
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59c5f187
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59c5f187
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59c5f187
Branch: refs/heads/trunk
Commit: 59c5f18784121f04030d1d0982f2e2285688ee11
Parents: 3619ae3
Author: Chris Douglas <cd...@apache.org>
Authored: Wed Feb 1 10:19:36 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed Feb 1 10:19:36 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/BlockLocation.java | 7 +++++--
.../java/org/apache/hadoop/fs/FileStatus.java | 21 +++++++++++++++++--
.../hadoop/fs/permission/FsCreateModes.java | 1 +
.../hadoop/fs/permission/FsPermission.java | 22 +++++++++++++++++---
.../org/apache/hadoop/fs/TestFileStatus.java | 19 +++++++++++++++++
.../org/apache/hadoop/fs/HdfsBlockLocation.java | 17 +++++++++++++--
.../hdfs/protocol/FsPermissionExtension.java | 2 ++
7 files changed, 80 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
index 7811ef5..b8cad3a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import java.io.IOException;
+import java.io.Serializable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,7 +30,9 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class BlockLocation {
+public class BlockLocation implements Serializable {
+ private static final long serialVersionUID = 0x22986f6d;
+
private String[] hosts; // Datanode hostnames
private String[] cachedHosts; // Datanode hostnames with a cached replica
private String[] names; // Datanode IP:xferPort for accessing the block
@@ -303,4 +306,4 @@ public class BlockLocation {
}
return result.toString();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index 6a79768..72ca24f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.fs;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputValidation;
+import java.io.Serializable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -31,11 +34,14 @@ import org.apache.hadoop.io.Writable;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class FileStatus implements Writable, Comparable<FileStatus> {
+public class FileStatus implements Writable, Comparable<FileStatus>,
+ Serializable, ObjectInputValidation {
+
+ private static final long serialVersionUID = 0x13caeae8;
private Path path;
private long length;
- private boolean isdir;
+ private Boolean isdir;
private short block_replication;
private long blocksize;
private long modification_time;
@@ -387,4 +393,15 @@ public class FileStatus implements Writable, Comparable<FileStatus> {
sb.append("}");
return sb.toString();
}
+
+ @Override
+ public void validateObject() throws InvalidObjectException {
+ if (null == path) {
+ throw new InvalidObjectException("No Path in deserialized FileStatus");
+ }
+ if (null == isdir) {
+ throw new InvalidObjectException("No type in deserialized FileStatus");
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
index a1ed0d7..2bd6f1f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class FsCreateModes extends FsPermission {
+ private static final long serialVersionUID = 0x22986f6d;
private final FsPermission unmasked;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index fabfc12..56e19dc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.fs.permission;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputValidation;
+import java.io.Serializable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,8 +39,10 @@ import org.apache.hadoop.io.WritableFactory;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class FsPermission implements Writable {
+public class FsPermission implements Writable, Serializable,
+ ObjectInputValidation {
private static final Log LOG = LogFactory.getLog(FsPermission.class);
+ private static final long serialVersionUID = 0x2fe08564;
static final WritableFactory FACTORY = new WritableFactory() {
@Override
@@ -60,7 +65,7 @@ public class FsPermission implements Writable {
private FsAction useraction = null;
private FsAction groupaction = null;
private FsAction otheraction = null;
- private boolean stickyBit = false;
+ private Boolean stickyBit = false;
private FsPermission() {}
@@ -202,7 +207,7 @@ public class FsPermission implements Writable {
return this.useraction == that.useraction
&& this.groupaction == that.groupaction
&& this.otheraction == that.otheraction
- && this.stickyBit == that.stickyBit;
+ && this.stickyBit.booleanValue() == that.stickyBit.booleanValue();
}
return false;
}
@@ -377,6 +382,7 @@ public class FsPermission implements Writable {
}
private static class ImmutableFsPermission extends FsPermission {
+ private static final long serialVersionUID = 0x1bab54bd;
public ImmutableFsPermission(short permission) {
super(permission);
}
@@ -386,4 +392,14 @@ public class FsPermission implements Writable {
throw new UnsupportedOperationException();
}
}
+
+ @Override
+ public void validateObject() throws InvalidObjectException {
+ if (null == useraction || null == groupaction || null == otheraction) {
+ throw new InvalidObjectException("Invalid mode in FsPermission");
+ }
+ if (null == stickyBit) {
+ throw new InvalidObjectException("No sticky bit in FsPermission");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
index dd5279d..35f2bad 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
@@ -26,6 +26,8 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -216,6 +218,23 @@ public class TestFileStatus {
MTIME, ATIME, PERMISSION, OWNER, GROUP, symlink, PATH);
validateToString(fileStatus);
}
+
+ @Test
+ public void testSerializable() throws Exception {
+ Path p = new Path("uqsf://ybpnyubfg:8020/sbb/one/onm");
+ FsPermission perm = FsPermission.getFileDefault();
+ FileStatus stat = new FileStatus(4344L, false, 4, 512L << 20, 12345678L,
+ 87654321L, perm, "yak", "dingo", p);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
+ try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(stat);
+ }
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+ FileStatus deser = (FileStatus) ois.readObject();
+ assertEquals(stat, deser);
+ }
+ }
/**
* Validate the accessors for FileStatus.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
index eac3f96..2ee7f41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.fs;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -28,9 +32,10 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class HdfsBlockLocation extends BlockLocation {
+public class HdfsBlockLocation extends BlockLocation implements Serializable {
+ private static final long serialVersionUID = 0x7aecec92;
- private final LocatedBlock block;
+ private transient LocatedBlock block;
public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) {
// Initialize with data from passed in BlockLocation
@@ -41,4 +46,12 @@ public class HdfsBlockLocation extends BlockLocation {
public LocatedBlock getLocatedBlock() {
return block;
}
+
+ private void readObject(ObjectInputStream ois)
+ throws IOException, ClassNotFoundException {
+ ois.defaultReadObject();
+ // LocatedBlock is not Serializable
+ block = null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59c5f187/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
index f74472d..786bb58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
*/
@InterfaceAudience.Private
public class FsPermissionExtension extends FsPermission {
+ private static final long serialVersionUID = 0x13c298a4;
+
private final static short ACL_BIT = 1 << 12;
private final static short ENCRYPTED_BIT = 1 << 13;
private final boolean aclBit;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org