You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2013/08/20 20:07:48 UTC
svn commit: r1515906 - in
/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs:
FSDataInputStream.java FSInputStream.java SupportsZeroCopy.java
ZeroCopyCursor.java ZeroCopyUnavailableException.java
Author: wang
Date: Tue Aug 20 18:07:47 2013
New Revision: 1515906
URL: http://svn.apache.org/r1515906
Log:
HDFS-4953. Enable HDFS local reads via mmap. Contributed by Colin Patrick McCabe.
Added:
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java (with props)
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java (with props)
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java (with props)
Modified:
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java
Modified: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java?rev=1515906&r1=1515905&r2=1515906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java Tue Aug 20 18:07:47 2013
@@ -28,9 +28,9 @@ import org.apache.hadoop.classification.
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FSDataInputStream extends DataInputStream
- implements Seekable, PositionedReadable, Closeable,
- ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead {
-
+ implements Seekable, PositionedReadable, Closeable,
+ ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
+ SupportsZeroCopy {
public FSDataInputStream(InputStream in)
throws IOException {
super(in);
@@ -167,4 +167,15 @@ public class FSDataInputStream extends D
"support setting the drop-behind caching setting.");
}
}
+
+ @Override
+ public ZeroCopyCursor createZeroCopyCursor()
+ throws IOException, ZeroCopyUnavailableException {
+ try {
+ return ((SupportsZeroCopy)in).createZeroCopyCursor();
+ }
+ catch (ClassCastException e) {
+ throw new ZeroCopyUnavailableException(e);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java?rev=1515906&r1=1515905&r2=1515906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java Tue Aug 20 18:07:47 2013
@@ -18,9 +18,11 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.nio.ByteBuffer;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ZeroCopyUnavailableException;
/****************************************************************
* FSInputStream is a generic old InputStream with a little bit
@@ -30,7 +32,7 @@ import org.apache.hadoop.classification.
@InterfaceAudience.LimitedPrivate({"HDFS"})
@InterfaceStability.Unstable
public abstract class FSInputStream extends InputStream
- implements Seekable, PositionedReadable {
+ implements Seekable, PositionedReadable, SupportsZeroCopy {
/**
* Seek to the given offset from the start of the file.
* The next read() will be from that location. Can't
@@ -86,4 +88,11 @@ public abstract class FSInputStream exte
throws IOException {
readFully(position, buffer, 0, buffer.length);
}
+
+ @Override
+ public ZeroCopyCursor createZeroCopyCursor()
+ throws IOException, ZeroCopyUnavailableException {
+ throw new ZeroCopyUnavailableException("zero copy is not implemented " +
+ "for this filesystem type.");
+ }
}
Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java (added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java Tue Aug 20 18:07:47 2013
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Supports zero-copy reads.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface SupportsZeroCopy {
+ /**
+ * Get a zero-copy cursor to use for zero-copy reads.
+ *
+ * @throws IOException
+ * If there was an error creating the ZeroCopyCursor
+ * @throws UnsupportedOperationException
+ * If this stream does not support zero-copy reads.
+ * This is used, for example, when one stream wraps another
+ * which may or may not support ZCR.
+ */
+ public ZeroCopyCursor createZeroCopyCursor()
+ throws IOException, ZeroCopyUnavailableException;
+}
\ No newline at end of file
Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java (added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java Tue Aug 20 18:07:47 2013
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * A ZeroCopyCursor allows you to make zero-copy reads.
+ *
+ * Cursors should be closed when they are no longer needed.
+ *
+ * Example:
+ * FSDataInputStream fis = fs.open("/file");
+ * ZeroCopyCursor cursor = fis.createZeroCopyCursor();
+ * try {
+ * cursor.read(128);
+ * ByteBuffer data = cursor.getData();
+ * processData(data);
+ * } finally {
+ * cursor.close();
+ * }
+ */
+public interface ZeroCopyCursor extends Closeable {
+ /**
+ * Set the fallback buffer used for this zero copy cursor.
+ * The fallback buffer is used when a true zero-copy read is impossible.
+ * If there is no fallback buffer, UnsupportedOperationException is thrown
+ * when a true zero-copy read cannot be done.
+ *
+ * @param fallbackBuffer The fallback buffer to set, or null for none.
+ */
+ public void setFallbackBuffer(ByteBuffer fallbackBuffer);
+
+ /**
+ * @return the fallback buffer in use, or null if there is none.
+ */
+ public ByteBuffer getFallbackBuffer();
+
+ /**
+ * @param skipChecksums Whether we should skip checksumming with this
+ * zero copy cursor.
+ */
+ public void setSkipChecksums(boolean skipChecksums);
+
+ /**
+ * @return Whether we should skip checksumming with this
+ * zero copy cursor.
+ */
+ public boolean getSkipChecksums();
+
+ /**
+ * @param allowShortReads Whether we should allow short reads.
+ */
+ public void setAllowShortReads(boolean allowShortReads);
+
+ /**
+ * @return Whether we should allow short reads.
+ */
+ public boolean getAllowShortReads();
+
+ /**
+ * Perform a zero-copy read.
+ *
+ * @param toRead The minimum number of bytes to read.
+ * Must not be negative. If we hit EOF before
+ * reading this many bytes, we will either throw
+ * EOFException (if allowShortReads = false), or
+ * return a short read (if allowShortReads = true).
+ * A short read could be as short as 0 bytes.
+ * @throws UnsupportedOperationException
+ * If a true zero-copy read cannot be done, and no fallback
+ * buffer was set.
+ * @throws EOFException
+ * If allowShortReads = false, and we can't read all the bytes
+ * that were requested. This will never be thrown if
+ * allowShortReads = true.
+ * @throws IOException
+ * If there was an error while reading the data.
+ */
+ public void read(int toRead)
+ throws UnsupportedOperationException, EOFException, IOException;
+
+ /**
+ * Get the current data buffer.
+ *
+ * This buffer will remain valid until either this cursor is closed, or we
+ * call read() again on this same cursor. You can find the amount of data
+ * that was read previously by calling ByteBuffer#remaining.
+ *
+ * @return The current data buffer.
+ */
+ public ByteBuffer getData();
+}
Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java (added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java Tue Aug 20 18:07:47 2013
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+public class ZeroCopyUnavailableException extends IOException {
+ private static final long serialVersionUID = 0L;
+
+ public ZeroCopyUnavailableException(String message) {
+ super(message);
+ }
+
+ public ZeroCopyUnavailableException(String message, Exception e) {
+ super(message, e);
+ }
+
+ public ZeroCopyUnavailableException(Exception e) {
+ super(e);
+ }
+}
Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
------------------------------------------------------------------------------
svn:eol-style = native