You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/11/02 17:30:16 UTC
[25/50] [abbrv] hadoop git commit: HDFS-7878. API - expose a unique
file identifier.
HDFS-7878. API - expose a unique file identifier.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d015e0bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d015e0bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d015e0bb
Branch: refs/heads/YARN-1011
Commit: d015e0bbd5416943cb4875274e67b7077c00e54b
Parents: 8122543
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Oct 31 09:44:01 2017 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Oct 31 09:44:01 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 46 ++++
.../org/apache/hadoop/fs/FilterFileSystem.java | 12 +
.../org/apache/hadoop/fs/HarFileSystem.java | 14 ++
.../main/java/org/apache/hadoop/fs/Options.java | 180 ++++++++++++++
.../java/org/apache/hadoop/fs/PathHandle.java | 50 ++++
.../org/apache/hadoop/fs/RawPathHandle.java | 119 +++++++++
.../src/site/markdown/filesystem/filesystem.md | 115 +++++++++
.../fs/contract/AbstractContractOpenTest.java | 247 ++++++++++++++++++-
.../hadoop/fs/contract/ContractOptions.java | 10 +
.../hadoop/fs/contract/ContractTestUtils.java | 32 +++
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 21 ++
.../hadoop/hdfs/DistributedFileSystem.java | 54 ++++
.../hadoop/hdfs/protocol/HdfsConstants.java | 4 +
.../hadoop/hdfs/protocol/HdfsPathHandle.java | 98 ++++++++
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 17 +-
.../src/main/proto/hdfs.proto | 7 +
.../hdfs/server/namenode/FSDirectory.java | 10 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 16 ++
.../hdfs/TestFileStatusSerialization.java | 78 ++++--
.../src/test/resources/contract/hdfs.xml | 12 +-
20 files changed, 1102 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index d43e41d..64021ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@@ -951,6 +952,51 @@ public abstract class FileSystem extends Configured implements Closeable {
}
/**
+ * Open an FSDataInputStream matching the PathHandle instance. The
+ * implementation may encode metadata in PathHandle to address the
+ * resource directly and verify that the resource referenced
+ * satisfies constraints specified at its construciton.
+ * @param fd PathHandle object returned by the FS authority.
+ * @param bufferSize the size of the buffer to use
+ * @throws IOException IO failure
+ * @throws UnsupportedOperationException If not overridden by subclass
+ */
+ public FSDataInputStream open(PathHandle fd, int bufferSize)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Create a durable, serializable handle to the referent of the given
+ * entity.
+ * @param stat Referent in the target FileSystem
+ * @param opt If absent, assume {@link HandleOpt#path()}.
+ * @throws IllegalArgumentException If the FileStatus does not belong to
+ * this FileSystem
+ * @throws UnsupportedOperationException If
+ * {@link #createPathHandle(FileStatus, HandleOpt[])}
+ * not overridden by subclass.
+ * @throws UnsupportedOperationException If this FileSystem cannot enforce
+ * the specified constraints.
+ */
+ public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
+ if (null == opt || 0 == opt.length) {
+ return createPathHandle(stat, HandleOpt.path());
+ }
+ return createPathHandle(stat, opt);
+ }
+
+ /**
+ * Hook to implement support for {@link PathHandle} operations.
+ * @param stat Referent in the target FileSystem
+ * @param opt Constraints that determine the validity of the
+ * {@link PathHandle} reference.
+ */
+ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
* Create an FSDataOutputStream at the indicated Path.
* Files are overwritten by default.
* @param f the file to create
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 954a041..f9bbfb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
@@ -164,6 +165,17 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public FSDataInputStream open(PathHandle fd, int bufferSize)
+ throws IOException {
+ return fs.open(fd, bufferSize);
+ }
+
+ @Override
+ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opts) {
+ return fs.getPathHandle(stat, opts);
+ }
+
+ @Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
return fs.append(f, bufferSize, progress);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index 4c2fd1b..f6ffcb4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.Options.HandleOpt;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.LineReader;
@@ -699,6 +700,19 @@ public class HarFileSystem extends FileSystem {
hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
}
+ @Override
+ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opts) {
+ // har consistency managed through metadata cache
+ // could extend HarMetaData to track more explicitly
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FSDataInputStream open(PathHandle fd, int bufferSize)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
/**
* Used for delegation token related functionality. Must delegate to
* underlying file system.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 0ac53b2..550e6b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.fs;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.BiFunction;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -325,4 +329,180 @@ public final class Options {
return processChecksumOpt(defaultOpt, userOpt, -1);
}
}
+
+ /**
+ * Options for creating {@link PathHandle} references.
+ */
+ public static class HandleOpt {
+ protected HandleOpt() {
+ }
+
+ /**
+ * Utility function for mapping
+ * {@link FileSystem#getPathHandle(FileStatus, HandleOpt[])} to a
+ * fixed set of handle options.
+ * @param fs Target filesystem
+ * @param opt Options to bind in partially evaluated function
+ * @return Function reference with options fixed
+ */
+ public static Function<FileStatus, PathHandle> resolve(
+ FileSystem fs, HandleOpt... opt) {
+ return resolve(fs::getPathHandle, opt);
+ }
+
+ /**
+ * Utility function for partial evaluation of {@link FileStatus}
+ * instances to a fixed set of handle options.
+ * @param fsr Function reference
+ * @param opt Options to associate with {@link FileStatus} instances to
+ * produce {@link PathHandle} instances.
+ * @return Function reference with options fixed
+ */
+ public static Function<FileStatus, PathHandle> resolve(
+ BiFunction<FileStatus, HandleOpt[], PathHandle> fsr,
+ HandleOpt... opt) {
+ return (stat) -> fsr.apply(stat, opt);
+ }
+
+ /**
+ * Handle is valid iff the referent is neither moved nor changed.
+ * Equivalent to changed(false), moved(false).
+ * @return Options requiring that the content and location of the entity
+ * be unchanged between calls.
+ */
+ public static HandleOpt[] exact() {
+ return new HandleOpt[] {changed(false), moved(false) };
+ }
+
+ /**
+ * Handle is valid iff the content of the referent is the same.
+ * Equivalent to changed(false), moved(true).
+ * @return Options requiring that the content of the entity is unchanged,
+ * but it may be at a different location.
+ */
+ public static HandleOpt[] content() {
+ return new HandleOpt[] {changed(false), moved(true) };
+ }
+
+ /**
+ * Handle is valid iff the referent is unmoved in the namespace.
+ * Equivalent to changed(true), moved(false).
+ * @return Options requiring that the referent exist in the same location,
+ * but its content may have changed.
+ */
+ public static HandleOpt[] path() {
+ return new HandleOpt[] {changed(true), moved(false) };
+ }
+
+ /**
+ * Handle is valid iff the referent exists in the namespace.
+ * Equivalent to changed(true), moved(true).
+ * @return Options requiring that the implementation resolve a reference
+ * to this entity regardless of changes to content or location.
+ */
+ public static HandleOpt[] reference() {
+ return new HandleOpt[] {changed(true), moved(true) };
+ }
+
+ /**
+ * @param allow If true, resolve references to this entity even if it has
+ * been modified.
+ * @return Handle option encoding parameter.
+ */
+ public static Data changed(boolean allow) {
+ return new Data(allow);
+ }
+
+ /**
+ * @param allow If true, resolve references to this entity anywhere in
+ * the namespace.
+ * @return Handle option encoding parameter.
+ */
+ public static Location moved(boolean allow) {
+ return new Location(allow);
+ }
+
+ /**
+ * Utility method to extract a HandleOpt from the set provided.
+ * @param c Target class
+ * @param opt List of options
+ * @param <T> Type constraint for exact match
+ * @throws IllegalArgumentException If more than one matching type is found.
+ * @return An option assignable from the specified type or null if either
+ * opt is null or a suitable match is not found.
+ */
+ public static <T extends HandleOpt> Optional<T> getOpt(
+ Class<T> c, HandleOpt... opt) {
+ if (null == opt) {
+ return Optional.empty();
+ }
+ T ret = null;
+ for (HandleOpt o : opt) {
+ if (c.isAssignableFrom(o.getClass())) {
+ if (ret != null) {
+ throw new IllegalArgumentException("Duplicate option "
+ + c.getSimpleName());
+ }
+
+ @SuppressWarnings("unchecked")
+ T tmp = (T) o;
+ ret = tmp;
+ }
+ }
+ return Optional.ofNullable(ret);
+ }
+
+ /**
+ * Option storing standard constraints on data.
+ */
+ public static class Data extends HandleOpt {
+ private final boolean allowChanged;
+ Data(boolean allowChanged) {
+ this.allowChanged = allowChanged;
+ }
+
+ /**
+ * Tracks whether any changes to file content are permitted.
+ * @return True if content changes are allowed, false otherwise.
+ */
+ public boolean allowChange() {
+ return allowChanged;
+ }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("data(allowChange=")
+ .append(allowChanged).append(")");
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Option storing standard constraints on location.
+ */
+ public static class Location extends HandleOpt {
+ private final boolean allowChanged;
+ Location(boolean allowChanged) {
+ this.allowChanged = allowChanged;
+ }
+
+ /**
+ * Tracks whether any changes to file location are permitted.
+ * @return True if relocation in the namespace is allowed, false
+ * otherwise.
+ */
+ public boolean allowChange() {
+ return allowChanged;
+ }
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("loc(allowChange=")
+ .append(allowChanged).append(")");
+ return sb.toString();
+ }
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathHandle.java
new file mode 100644
index 0000000..60aa6a5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathHandle.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Opaque, serializable reference to an entity in the FileSystem. May contain
+ * metadata sufficient to resolve or verify subsequent accesses indepedent of
+ * other modifications to the FileSystem.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface PathHandle extends Serializable {
+
+ /**
+ * @return Serialized from in bytes.
+ */
+ default byte[] toByteArray() {
+ ByteBuffer bb = bytes();
+ byte[] ret = new byte[bb.remaining()];
+ bb.get(ret);
+ return ret;
+ }
+
+ ByteBuffer bytes();
+
+ @Override
+ boolean equals(Object other);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawPathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawPathHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawPathHandle.java
new file mode 100644
index 0000000..dc735f4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawPathHandle.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamException;
+import java.nio.ByteBuffer;
+
+/**
+ * Generic format of FileStatus objects. When the origin is unknown, the
+ * attributes of the handle are undefined.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class RawPathHandle implements PathHandle {
+
+ private static final long serialVersionUID = 0x12ba4689510L;
+
+ public static final int MAX_SIZE = 1 << 20;
+
+ private transient ByteBuffer fd;
+
+ /**
+ * Store a reference to the given bytes as the serialized form.
+ * @param fd serialized bytes
+ */
+ public RawPathHandle(ByteBuffer fd) {
+ this.fd = null == fd
+ ? ByteBuffer.allocate(0)
+ : fd.asReadOnlyBuffer();
+ }
+
+ /**
+ * Initialize using a copy of bytes from the serialized handle.
+ * @param handle PathHandle to preserve in serialized form.
+ */
+ public RawPathHandle(PathHandle handle) {
+ ByteBuffer hb = null == handle
+ ? ByteBuffer.allocate(0)
+ : handle.bytes();
+ fd = ByteBuffer.allocate(hb.remaining());
+ fd.put(hb);
+ fd.flip();
+ }
+
+ @Override
+ public ByteBuffer bytes() {
+ return fd.asReadOnlyBuffer();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PathHandle)) {
+ return false;
+ }
+ PathHandle o = (PathHandle) other;
+ return bytes().equals(o.bytes());
+ }
+
+ @Override
+ public int hashCode() {
+ return bytes().hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return bytes().toString();
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.defaultWriteObject();
+ out.writeInt(fd.remaining());
+ if (fd.hasArray()) {
+ out.write(fd.array(), fd.position(), fd.remaining());
+ } else {
+ byte[] x = new byte[fd.remaining()];
+ fd.slice().get(x);
+ out.write(x);
+ }
+ }
+
+ private void readObject(ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ int len = in.readInt();
+ if (len < 0 || len > MAX_SIZE) {
+ throw new IOException("Illegal buffer length " + len);
+ }
+ byte[] x = new byte[len];
+ in.readFully(x);
+ fd = ByteBuffer.wrap(x);
+ }
+
+ private void readObjectNoData() throws ObjectStreamException {
+ throw new InvalidObjectException("Stream data required");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index e67cbe3..d251804 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -702,6 +702,121 @@ symbolic links
exists in the metadata, but no copies of any its blocks can be located;
-`FileNotFoundException` would seem more accurate and useful.
+### `PathHandle getPathHandle(FileStatus stat, HandleOpt... options)`
+
+Implementaions without a compliant call MUST throw `UnsupportedOperationException`
+
+#### Preconditions
+
+ let stat = getFileStatus(Path p)
+ let FS' where:
+ (FS.Directories', FS.Files', FS.Symlinks')
+ p' in paths(FS') where:
+ exists(FS, stat.path) implies exists(FS', p')
+
+The referent of a `FileStatus` instance, at the time it was resolved, is the
+same referent as the result of `getPathHandle(FileStatus)`. The `PathHandle`
+may be used in subsequent operations to ensure invariants hold between
+calls.
+
+The `options` parameter specifies whether a subsequent call e.g.,
+`open(PathHandle)` will succeed if the referent data or location changed. By
+default, any modification results in an error. The caller MAY specify
+relaxations that allow operations to succeed even if the referent exists at
+a different path and/or its data are changed.
+
+An implementation MUST throw `UnsupportedOperationException` if it cannot
+support the semantics specified by the caller. The default set of options
+are as follows.
+
+| | Unmoved | Moved |
+|-----------:|:--------:|:---------:|
+| Unchanged | EXACT | CONTENT |
+| Changed | PATH | REFERENCE |
+
+Changes to ownership, extended attributes, and other metadata are not
+required to match the `PathHandle`. Implementations can extend the set of
+`HandleOpt` parameters with custom constraints.
+
+##### Examples
+
+A client specifies that the `PathHandle` should track the entity across
+renames using `REFERENCE`. The implementation MUST throw an
+`UnsupportedOperationException` when creating the `PathHandle` unless
+failure to resolve the reference implies the entity no longer exists.
+
+A client specifies that the `PathHandle` should resolve only if the entity
+is unchanged using `PATH`. The implementation MUST throw an
+`UnsupportedOperationException` when creating the `PathHandle` unless it can
+distinguish between an identical entity located subsequently at the same
+path.
+
+#### Postconditions
+
+ result = PathHandle(p')
+
+#### Implementation notes
+
+The referent of a `PathHandle` is the namespace when the `FileStatus`
+instance was created, _not_ its state when the `PathHandle` is created. An
+implementation MAY reject attempts to create or resolve `PathHandle`
+instances that are valid, but expensive to service.
+
+Object stores that implement rename by copying objects MUST NOT claim to
+support `CONTENT` and `REFERENCE` unless the lineage of the object is
+resolved.
+
+It MUST be possible to serialize a `PathHandle` instance and reinstantiate
+it in one or more processes, on another machine, and arbitrarily far into
+the future without changing its semantics. The implementation MUST refuse to
+resolve instances if it can no longer guarantee its invariants.
+
+#### HDFS implementation notes
+
+HDFS does not support `PathHandle` references to directories or symlinks.
+Support for `CONTENT` and `REFERENCE` looks up files by INode. INodes are
+not unique across NameNodes, so federated clusters SHOULD include enough
+metadata in the `PathHandle` to detect references from other namespaces.
+
+### `FSDataInputStream open(PathHandle handle, int bufferSize)`
+
+Implementaions without a compliant call MUST throw `UnsupportedOperationException`
+
+#### Preconditions
+
+ let fd = getPathHandle(FileStatus stat)
+ if stat.isdir : raise IOException
+ let FS' where:
+ (FS.Directories', FS.Files', FS.Symlinks')
+ p' in FS.Files' where:
+ FS.Files'[p'] = fd
+ if not exists(FS', p') : raise FileNotFoundException
+
+The implementation MUST resolve the referent of the `PathHandle` following
+the constraints specified at its creation by `getPathHandle(FileStatus)`.
+
+Metadata necessary for the `FileSystem` to satisfy this contract MAY be
+encoded in the `PathHandle`.
+
+#### Postconditions
+
+ result = FSDataInputStream(0, FS.Files'[p'])
+
+The stream returned is subject to the constraints of a stream returned by
+`open(Path)`. Constraints checked on open MAY hold to hold for the stream, but
+this is not guaranteed.
+
+For example, a `PathHandle` created with `CONTENT` constraints MAY return a
+stream that ignores updates to the file after it is opened, if it was
+unmodified when `open(PathHandle)` was resolved.
+
+#### Implementation notes
+
+An implementation MAY check invariants either at the server or before
+returning the stream to the client. For example, an implementation may open
+the file, then verify the invariants in the `PathHandle` using
+`getFileStatus(Path)` to implement `CONTENT`. This could yield false
+positives and it requires additional RPC traffic.
### `boolean delete(Path p, boolean recursive)`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
index ccf188f..9b6bbd4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
@@ -15,29 +15,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.fs.contract;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Options.HandleOpt;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.fs.RawPathHandle;
import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.appendFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyRead;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyFileContents;
+
+import org.junit.Test;
/**
* Test Seek operations
*/
-public abstract class AbstractContractOpenTest extends AbstractFSContractTestBase {
+public abstract class AbstractContractOpenTest
+ extends AbstractFSContractTestBase {
private FSDataInputStream instream;
@@ -163,5 +172,229 @@ public abstract class AbstractContractOpenTest extends AbstractFSContractTestBas
instream.close();
}
+ /**
+ * Skip a test case if the FS doesn't support file references.
+ * The feature is assumed to be unsupported unless stated otherwise.
+ */
+ protected void assumeSupportsFileReference() throws IOException {
+ if (getContract().isSupported(SUPPORTS_FILE_REFERENCE, false)) {
+ return;
+ }
+ skip("Skipping as unsupported feature: " + SUPPORTS_FILE_REFERENCE);
+ }
+
+ /**
+ * Skip a test case if the FS doesn't support content validation.
+ * The feature is assumed to be unsupported unless stated otherwise.
+ */
+ protected void assumeSupportsContentCheck() throws IOException {
+ if (getContract().isSupported(SUPPORTS_CONTENT_CHECK, false)) {
+ return;
+ }
+ skip("Skipping as unsupported feature: " + SUPPORTS_CONTENT_CHECK);
+ }
+
+ private PathHandle getHandleOrSkip(FileStatus stat, HandleOpt... opts) {
+ try {
+ return getFileSystem().getPathHandle(stat, opts);
+ } catch (UnsupportedOperationException e) {
+ skip("FileSystem does not support " + Arrays.toString(opts));
+ }
+ // unreachable
+ return null;
+ }
+
+ /**
+ * Verify {@link HandleOpt#exact()} handle semantics.
+ * @throws Throwable on error
+ */
+ @Test
+ public void testOpenFileByExact() throws Throwable {
+ describe("verify open(getPathHandle(FileStatus, exact())) operations" +
+ "detect changes");
+ assumeSupportsContentCheck();
+ assumeSupportsFileReference();
+ Path path1 = path("testopenfilebyexact1");
+ Path path2 = path("testopenfilebyexact2");
+ byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+ createFile(getFileSystem(), path1, false, file1);
+ FileStatus stat = getFileSystem().getFileStatus(path1);
+ assertNotNull(stat);
+ assertEquals(path1, stat.getPath());
+ ContractTestUtils.rename(getFileSystem(), path1, path2);
+ // create identical file at same location, orig still exists at path2
+ createFile(getFileSystem(), path1, false, file1);
+
+ PathHandle fd = getHandleOrSkip(stat, HandleOpt.exact());
+
+ // verify path1, path2 contents identical
+ verifyFileContents(getFileSystem(), path1, file1);
+ verifyFileContents(getFileSystem(), path2, file1);
+ try {
+ // the PathHandle will not resolve, even though
+ // the original entity exists, it has not been modified, and an
+ // identical file exists at the old path. The handle would also
+ // fail to resolve if path1 had been modified
+ instream = getFileSystem().open(fd, 1 << 15);
+ fail("Expected an exception");
+ } catch (IOException e) {
+ // expected
+ }
+ }
+
+ /**
+ * Verify {@link HandleOpt#content()} handle semantics.
+ * @throws Throwable on error
+ */
+ @Test
+ public void testOpenFileByContent() throws Throwable {
+ describe("verify open(getPathHandle(FileStatus, content())) operations" +
+ "follow relocation");
+ assumeSupportsContentCheck();
+ assumeSupportsFileReference();
+ Path path1 = path("testopenfilebycontent1");
+ Path path2 = path("testopenfilebycontent2");
+ byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+ createFile(getFileSystem(), path1, false, file1);
+ FileStatus stat = getFileSystem().getFileStatus(path1);
+ assertNotNull(stat);
+ assertEquals(path1, stat.getPath());
+ // rename the file after obtaining FileStatus
+ ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+ // obtain handle to entity from #getFileStatus call
+ PathHandle fd = getHandleOrSkip(stat, HandleOpt.content());
+
+ try (FSDataInputStream in = getFileSystem().open(fd, 1 << 15)) {
+ // verify read of consistent content at new location
+ verifyRead(in, file1, 0, TEST_FILE_LEN);
+ }
+
+ // modify the file at its new location by appending data
+ byte[] file1a = dataset(TEST_FILE_LEN, 44, 255);
+ appendFile(getFileSystem(), path2, file1a);
+ byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
+ System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
+ // verify fd entity contains contents of file1 + appended bytes
+ verifyFileContents(getFileSystem(), path2, file1x);
+
+ try {
+ // handle should not resolve when content changed
+ instream = getFileSystem().open(fd, 1 << 15);
+ fail("Failed to detect change to content");
+ } catch (IOException e) {
+ // expected
+ }
+ }
+
+
+ /**
+ * Verify {@link HandleOpt#path()} handle semantics.
+ * @throws Throwable on error
+ */
+ @Test
+ public void testOpenFileByPath() throws Throwable {
+ describe("verify open(getPathHandle(FileStatus, path())) operations" +
+ "detect changes");
+ assumeSupportsContentCheck();
+ Path path1 = path("testopenfilebypath1");
+ Path path2 = path("testopenfilebypath2");
+
+ byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+ createFile(getFileSystem(), path1, false, file1);
+ FileStatus stat = getFileSystem().getFileStatus(path1);
+ assertNotNull(stat);
+ assertEquals(path1, stat.getPath());
+ ContractTestUtils.rename(getFileSystem(), path1, path2);
+ // create identical file at same location, orig still exists at path2
+ createFile(getFileSystem(), path1, false, file1);
+
+ PathHandle fd = getHandleOrSkip(stat, HandleOpt.path());
+
+ // verify path1, path2 contents identical
+ verifyFileContents(getFileSystem(), path1, file1);
+ verifyFileContents(getFileSystem(), path2, file1);
+ try {
+ // verify attempt to resolve the handle fails
+ instream = getFileSystem().open(fd, 1 << 15);
+ fail("Expected an exception");
+ } catch (IOException e) {
+ // expected
+ }
+ }
+
+ /**
+ * Verify {@link HandleOpt#reference()} handle semantics.
+ * @throws Throwable on error
+ */
+ @Test
+ public void testOpenFileByReference() throws Throwable {
+ describe("verify open(getPathHandle(FileStatus, reference())) operations" +
+ " are independent of rename");
+ assumeSupportsFileReference();
+ Path path1 = path("testopenfilebyref1");
+ Path path2 = path("testopenfilebyref2");
+
+ byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+ createFile(getFileSystem(), path1, false, file1);
+ FileStatus stat = getFileSystem().getFileStatus(path1);
+ assertNotNull(stat);
+ assertEquals(path1, stat.getPath());
+ ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+ byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
+ createFile(getFileSystem(), path1, false, file2);
+ byte[] file1a = dataset(TEST_FILE_LEN, 42, 255);
+ appendFile(getFileSystem(), path2, file1a);
+ byte[] file1x = Arrays.copyOf(file1, file1.length + file1a.length);
+ System.arraycopy(file1a, 0, file1x, file1.length, file1a.length);
+
+ PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
+
+ // verify path2 contains contents of file1 + appended bytes
+ verifyFileContents(getFileSystem(), path2, file1x);
+ // verify path1 contents contents of file2
+ verifyFileContents(getFileSystem(), path1, file2);
+
+ // verify fd contains contents of file1 + appended bytes
+ instream = getFileSystem().open(fd, 1 << 15);
+ verifyRead(instream, file1x, 0, TEST_FILE_LEN);
+ }
+
+ /**
+ * Verify {@link PathHandle} may be serialized and restored.
+ * @throws Throwable on error
+ */
+ @Test
+ public void testOpenFileBySerializedReference() throws Throwable {
+ describe("verify PathHandle supports generic serialization");
+ assumeSupportsFileReference();
+ Path path1 = path("testopenfilebyref1");
+ Path path2 = path("testopenfilebyref2");
+
+ byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
+ createFile(getFileSystem(), path1, false, file1);
+ FileStatus stat = getFileSystem().getFileStatus(path1);
+ assertNotNull(stat);
+ assertEquals(path1, stat.getPath());
+ ContractTestUtils.rename(getFileSystem(), path1, path2);
+
+ byte[] file2 = dataset(TEST_FILE_LEN, 44, 255);
+ createFile(getFileSystem(), path1, false, file2);
+
+ PathHandle fd = getHandleOrSkip(stat, HandleOpt.reference());
+
+ // serialize PathHandle
+ ByteBuffer sb = fd.bytes();
+ PathHandle fdb = new RawPathHandle(sb);
+
+ instream = getFileSystem().open(fdb, 1 << 15);
+ // verify stat contains contents of file1
+ verifyRead(instream, file1, 0, TEST_FILE_LEN);
+ // verify path2 contains contents of file1
+ verifyFileContents(getFileSystem(), path2, file1);
+ // verify path1 contents contents of file2
+ verifyFileContents(getFileSystem(), path1, file2);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
index b778f0d..cca3d4c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java
@@ -192,6 +192,16 @@ public interface ContractOptions {
String SUPPORTS_POSITIONED_READABLE = "supports-positioned-readable";
/**
+ * Indicates that FS exposes durable references to files.
+ */
+ String SUPPORTS_FILE_REFERENCE = "supports-file-reference";
+
+ /**
+ * Indicates that FS supports content checks on open.
+ */
+ String SUPPORTS_CONTENT_CHECK = "supports-content-check";
+
+ /**
* Maximum path length
* {@value}
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index 4dd9c3f..e0cc7d6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -403,6 +403,21 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Rename operation. Safety check for attempts to rename the root directory.
+ * Verifies that src no longer exists after rename.
+ * @param fileSystem filesystem to work with
+ * @param src source path
+ * @param dst destination path
+ * @throws IOException If rename fails or src is the root directory.
+ */
+ public static void rename(FileSystem fileSystem, Path src, Path dst)
+ throws IOException {
+ rejectRootOperation(src, false);
+ assertTrue(fileSystem.rename(src, dst));
+ assertPathDoesNotExist(fileSystem, "renamed", src);
+ }
+
+ /**
* Block any operation on the root path. This is a safety check
* @param path path in the filesystem
* @param allowRootOperation can the root directory be manipulated?
@@ -623,6 +638,23 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Append to an existing file.
+ * @param fs filesystem
+ * @param path path to file
+ * @param data data to append. Can be null
+ * @throws IOException On any error
+ */
+ public static void appendFile(FileSystem fs,
+ Path path,
+ byte[] data) throws IOException {
+ try (FSDataOutputStream stream = fs.appendFile(path).build()) {
+ if (data != null && data.length > 0) {
+ stream.write(data);
+ }
+ }
+ }
+
+ /**
* Touch a file.
* @param fs filesystem
* @param path path
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 24235a1..2a8bf0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -862,4 +862,25 @@ public class DFSUtilClient {
}
return threadPoolExecutor;
}
+
+ private static final int INODE_PATH_MAX_LENGTH = 3 * Path.SEPARATOR.length()
+ + HdfsConstants.DOT_RESERVED_STRING.length()
+ + HdfsConstants.DOT_INODES_STRING.length()
+ + (int)Math.ceil(Math.log10(Long.MAX_VALUE)) + 1;
+
+ /**
+ * Create the internal unique file path from HDFS file ID (inode ID). Unlike
+ * a regular file path, this one is guaranteed to refer to the same file at
+ * all times, across overwrites, etc.
+ * @param fileId File ID.
+ * @return The internal ID-based path.
+ */
+ public static Path makePathFromFileId(long fileId) {
+ StringBuilder sb = new StringBuilder(INODE_PATH_MAX_LENGTH);
+ sb.append(Path.SEPARATOR).append(HdfsConstants.DOT_RESERVED_STRING)
+ .append(Path.SEPARATOR).append(HdfsConstants.DOT_INODES_STRING)
+ .append(Path.SEPARATOR).append(fileId);
+ return new Path(sb.toString());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index a8a5cfa..2a6bd21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -44,9 +44,11 @@ import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
+import org.apache.hadoop.fs.PathHandle;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.Options.HandleOpt;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.QuotaUsage;
@@ -81,6 +83,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
@@ -105,6 +108,7 @@ import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
/****************************************************************
* Implementation of the abstract FileSystem for the DFS system.
@@ -319,6 +323,56 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ /**
+ * Opens an FSDataInputStream with the indicated file ID extracted from
+ * the {@link PathHandle}.
+ * @param fd Reference to entity in this FileSystem.
+ * @param bufferSize the size of the buffer to be used.
+ */
+ @Override
+ public FSDataInputStream open(PathHandle fd, int bufferSize)
+ throws IOException {
+ if (!(fd instanceof HdfsPathHandle)) {
+ fd = new HdfsPathHandle(fd.bytes());
+ }
+ HdfsPathHandle id = (HdfsPathHandle) fd;
+ return open(DFSUtilClient.makePathFromFileId(id.getInodeId()), bufferSize);
+ }
+
+ /**
+ * Create a handle to an HDFS file.
+ * @param st HdfsFileStatus instance from NameNode
+ * @param opts Standard handle arguments
+ * @throws IllegalArgumentException If the FileStatus instance refers to a
+ * directory, symlink, or another namesystem.
+ * @throws UnsupportedOperationException If opts are not specified or both
+ * data and location are not allowed to change.
+ * @return A handle to the file.
+ */
+ @Override
+ protected PathHandle createPathHandle(FileStatus st, HandleOpt... opts) {
+ if (!(st instanceof HdfsFileStatus)) {
+ throw new IllegalArgumentException("Invalid FileStatus "
+ + st.getClass().getSimpleName());
+ }
+ if (st.isDirectory() || st.isSymlink()) {
+ throw new IllegalArgumentException("PathHandle only available for files");
+ }
+ if (!getUri().getAuthority().equals(st.getPath().toUri().getAuthority())) {
+ throw new IllegalArgumentException("Wrong FileSystem: " + st.getPath());
+ }
+ HandleOpt.Data data = HandleOpt.getOpt(HandleOpt.Data.class, opts)
+ .orElse(HandleOpt.changed(false));
+ HandleOpt.Location loc = HandleOpt.getOpt(HandleOpt.Location.class, opts)
+ .orElse(HandleOpt.moved(false));
+ if (!data.allowChange() || !loc.allowChange()) {
+ throw new UnsupportedOperationException("Unsupported opts "
+ + Arrays.stream(opts)
+ .map(HandleOpt::toString).collect(Collectors.joining(",")));
+ }
+ return new HdfsPathHandle((HdfsFileStatus)st);
+ }
+
@Override
public FSDataOutputStream append(Path f, final int bufferSize,
final Progressable progress) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 8c44293..8245d1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -58,6 +58,10 @@ public final class HdfsConstants {
= Path.SEPARATOR + DOT_SNAPSHOT_DIR;
public static final String SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR
= Path.SEPARATOR + DOT_SNAPSHOT_DIR + Path.SEPARATOR;
+ public final static String DOT_RESERVED_STRING = ".reserved";
+ public final static String DOT_RESERVED_PATH_PREFIX = Path.SEPARATOR
+ + DOT_RESERVED_STRING;
+ public final static String DOT_INODES_STRING = ".inodes";
/**
* Generation stamp of blocks that pre-date the introduction
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
new file mode 100644
index 0000000..2d8dcca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsPathHandleProto;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Opaque handle to an entity in HDFS.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class HdfsPathHandle implements PathHandle {
+
+ private static final long serialVersionUID = 0xc5308795428L;
+
+ private final long inodeId;
+
+ public HdfsPathHandle(HdfsFileStatus hstat) {
+ this(hstat.getFileId());
+ }
+
+ public HdfsPathHandle(long inodeId) {
+ this.inodeId = inodeId;
+ }
+
+ public HdfsPathHandle(ByteBuffer bytes) throws IOException {
+ if (null == bytes) {
+ throw new IOException("Missing PathHandle");
+ }
+ HdfsPathHandleProto p =
+ HdfsPathHandleProto.parseFrom(ByteString.copyFrom(bytes));
+ inodeId = p.getInodeId();
+ }
+
+ public long getInodeId() {
+ return inodeId;
+ }
+
+ @Override
+ public ByteBuffer bytes() {
+ return HdfsPathHandleProto.newBuilder()
+ .setInodeId(getInodeId())
+ .build()
+ .toByteString()
+ .asReadOnlyByteBuffer();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (null == other) {
+ return false;
+ }
+ if (!HdfsPathHandle.class.equals(other.getClass())) {
+ // require exact match
+ return false;
+ }
+ HdfsPathHandle o = (HdfsPathHandle)other;
+ return getInodeId() == o.getInodeId();
+ }
+
+ @Override
+ public int hashCode() {
+ return Long.hashCode(inodeId);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{ ");
+ sb.append("inodeId : ").append(Long.toString(getInodeId()));
+ sb.append(" }");
+ return sb.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 21b1e1a..cecdfad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -82,13 +82,14 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -159,6 +160,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsPathHandleProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
@@ -1561,6 +1563,19 @@ public class PBHelperClient {
return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
}
+ public static HdfsPathHandle convert(HdfsPathHandleProto fd) {
+ if (null == fd) {
+ return null;
+ }
+ return new HdfsPathHandle(fd.getInodeId());
+ }
+
+ public static HdfsPathHandleProto convert(HdfsPathHandle fd) {
+ return HdfsPathHandleProto.newBuilder()
+ .setInodeId(fd.getInodeId())
+ .build();
+ }
+
public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
if (fs == null) {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index c38a9be..953bf19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -397,6 +397,13 @@ message AddErasureCodingPolicyResponseProto {
}
/**
+ * Placeholder type for consistent HDFS operations.
+ */
+message HdfsPathHandleProto {
+ optional uint64 inodeId = 1;
+}
+
+/**
* Status of a file, directory or symlink
* Optionally includes a file's block locations if requested by client on the rpc call.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 4f181e6..d119457 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -122,14 +122,16 @@ public class FSDirectory implements Closeable {
@VisibleForTesting
static boolean CHECK_RESERVED_FILE_NAMES = true;
- public final static String DOT_RESERVED_STRING = ".reserved";
- public final static String DOT_RESERVED_PATH_PREFIX = Path.SEPARATOR
- + DOT_RESERVED_STRING;
+ public final static String DOT_RESERVED_STRING =
+ HdfsConstants.DOT_RESERVED_STRING;
+ public final static String DOT_RESERVED_PATH_PREFIX =
+ HdfsConstants.DOT_RESERVED_PATH_PREFIX;
public final static byte[] DOT_RESERVED =
DFSUtil.string2Bytes(DOT_RESERVED_STRING);
private final static String RAW_STRING = "raw";
private final static byte[] RAW = DFSUtil.string2Bytes(RAW_STRING);
- public final static String DOT_INODES_STRING = ".inodes";
+ public final static String DOT_INODES_STRING =
+ HdfsConstants.DOT_INODES_STRING;
public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING);
private final static byte[] DOT_DOT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index f811d3d..39f76a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -54,6 +54,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
@@ -1049,4 +1050,19 @@ public class TestDFSUtil {
DFSUtilClient.isHDFSEncryptionEnabled(conf));
}
+
+ @Test
+ public void testFileIdPath() throws Throwable {
+ // /.reserved/.inodes/
+ String prefix = Path.SEPARATOR + HdfsConstants.DOT_RESERVED_STRING +
+ Path.SEPARATOR + HdfsConstants.DOT_INODES_STRING +
+ Path.SEPARATOR;
+ Random r = new Random();
+ for (int i = 0; i < 100; ++i) {
+ long inode = r.nextLong() & Long.MAX_VALUE;
+ assertEquals(new Path(prefix + inode),
+ DFSUtilClient.makePathFromFileId(inode));
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
index e5d0566..42c9acf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.net.URI;
import org.apache.hadoop.fs.FSProtos.FileStatusProto;
@@ -33,7 +37,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
import com.google.protobuf.ByteString;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
/**
* Verify compatible FileStatus/HdfsFileStatus serialization.
@@ -53,6 +57,27 @@ public class TestFileStatusSerialization {
assertEquals(expected.getBlockSize(), actual.getBlockSize());
}
+ private static final URI BASEURI = new Path("hdfs://foobar").toUri();
+ private static final Path BASEPATH = new Path("/dingos");
+ private static final String FILE = "zot";
+ private static final Path FULLPATH = new Path("hdfs://foobar/dingos/zot");
+ private static HdfsFileStatusProto.Builder baseStatus() {
+ FsPermission perm = FsPermission.getFileDefault();
+ HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
+ .setFileType(FileType.IS_FILE)
+ .setPath(ByteString.copyFromUtf8("zot"))
+ .setLength(4344)
+ .setPermission(PBHelperClient.convert(perm))
+ .setOwner("hadoop")
+ .setGroup("unqbbc")
+ .setModificationTime(12345678L)
+ .setAccessTime(87654321L)
+ .setBlockReplication(10)
+ .setBlocksize(1L << 33)
+ .setFlags(0);
+ return hspb;
+ }
+
/**
* Test API backwards-compatibility with 2.x applications w.r.t. FsPermission.
*/
@@ -65,21 +90,12 @@ public class TestFileStatusSerialization {
// test verifies.
for (int i = 0; i < flagmask; ++i) {
FsPermission perm = FsPermission.createImmutable((short) 0013);
- HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
- .setFileType(FileType.IS_FILE)
- .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos/zot"))
- .setLength(4344)
+ HdfsFileStatusProto.Builder hspb = baseStatus()
.setPermission(PBHelperClient.convert(perm))
- .setOwner("hadoop")
- .setGroup("unqbbc")
- .setModificationTime(12345678L)
- .setAccessTime(87654321L)
- .setBlockReplication(10)
- .setBlocksize(1L << 33)
.setFlags(i);
HdfsFileStatus stat = PBHelperClient.convert(hspb.build());
- stat.makeQualified(new URI("hdfs://foobar"), new Path("/dingos"));
- assertEquals(new Path("hdfs://foobar/dingos/zot"), stat.getPath());
+ stat.makeQualified(BASEURI, BASEPATH);
+ assertEquals(FULLPATH, stat.getPath());
// verify deprecated FsPermissionExtension methods
FsPermission sp = stat.getPermission();
@@ -103,23 +119,29 @@ public class TestFileStatusSerialization {
assertEquals(sp.getErasureCodedBit(), fstat.isErasureCoded());
}
}
- // param for LocatedFileStatus, HttpFileStatus
+
+ @Test
+ public void testJavaSerialization() throws Exception {
+ HdfsFileStatusProto hsp = baseStatus().build();
+ HdfsFileStatus hs = PBHelperClient.convert(hsp);
+ hs.makeQualified(BASEURI, BASEPATH);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
+ try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(hs);
+ }
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ try (ObjectInputStream ois = new ObjectInputStream(bais)) {
+ FileStatus deser = (FileStatus) ois.readObject();
+ assertEquals(hs, deser);
+ checkFields(hs, deser);
+ }
+ }
@Test
public void testCrossSerializationProto() throws Exception {
- FsPermission perm = FsPermission.getFileDefault();
for (FileType t : FileType.values()) {
- HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
- .setFileType(t)
- .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos"))
- .setLength(4344)
- .setPermission(PBHelperClient.convert(perm))
- .setOwner("hadoop")
- .setGroup("unqbbc")
- .setModificationTime(12345678L)
- .setAccessTime(87654321L)
- .setBlockReplication(10)
- .setBlocksize(1L << 33);
+ HdfsFileStatusProto.Builder hspb = baseStatus()
+ .setFileType(t);
if (FileType.IS_SYMLINK.equals(t)) {
hspb.setSymlink(ByteString.copyFromUtf8("hdfs://yaks/dingos"));
}
@@ -146,7 +168,9 @@ public class TestFileStatusSerialization {
byte[] dst = fsp.toByteArray();
HdfsFileStatusProto hsp2 = HdfsFileStatusProto.parseFrom(dst);
assertEquals(hsp, hsp2);
- checkFields(PBHelperClient.convert(hsp), PBHelperClient.convert(hsp2));
+ FileStatus hstat = PBHelperClient.convert(hsp);
+ FileStatus hstat2 = PBHelperClient.convert(hsp2);
+ checkFields(hstat, hstat2);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d015e0bb/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
index 8555bd9..d38d3d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
@@ -101,4 +101,14 @@
<value>true</value>
</property>
-</configuration>
\ No newline at end of file
+ <property>
+ <name>fs.contract.supports-file-reference</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-content-check</name>
+ <value>false</value>
+ </property>
+
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org