You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/07/23 21:00:24 UTC
svn commit: r558832 - in /lucene/hadoop/branches/branch-0.14: ./
src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/
src/test/org/apache/hadoop/dfs/
Author: cutting
Date: Mon Jul 23 12:00:23 2007
New Revision: 558832
URL: http://svn.apache.org/viewvc?view=rev&rev=558832
Log:
Merge -r 558830:558831 from trunk to 0.14 branch. Fixes: HADOOP-1619.
Modified:
lucene/hadoop/branches/branch-0.14/CHANGES.txt
lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java
lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java
lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java
Modified: lucene/hadoop/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/CHANGES.txt?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.14/CHANGES.txt Mon Jul 23 12:00:23 2007
@@ -401,6 +401,9 @@
134. HADOOP-1632. Fix an IllegalArgumentException in fsck.
(Hairong Kuang via cutting)
+135. HADOOP-1619. Fix FSInputChecker to not attempt to read past EOF.
+ (Hairong Kuang via cutting)
+
Release 0.13.0 - 2007-06-08
Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java Mon Jul 23 12:00:23 2007
@@ -1264,7 +1264,12 @@
public long skip(long n) throws IOException {
if ( n > 0 ) {
- seek(getPos()+n);
+ long curPos = getPos();
+ long fileLen = getFileLength();
+ if( n+curPos > fileLen ) {
+ n = fileLen - curPos;
+ }
+ seek(curPos+n);
return n;
}
return n < 0 ? -1 : 0;
Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Mon Jul 23 12:00:23 2007
@@ -96,6 +96,7 @@
private static final int HEADER_LENGTH = 8;
private int bytesPerSum = 1;
+ private long fileLen = -1L;
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
throws IOException {
@@ -202,6 +203,57 @@
}
return nread;
}
+
+ /* Return the file length */
+ private long getFileLength() throws IOException {
+ if( fileLen==-1L ) {
+ fileLen = fs.getContentLength(file);
+ }
+ return fileLen;
+ }
+
+ /**
+ * Skips over and discards <code>n</code> bytes of data from the
+ * input stream.
+ *
+ *The <code>skip</code> method skips over some smaller number of bytes
+ * when reaching end of file before <code>n</code> bytes have been skipped.
+ * The actual number of bytes skipped is returned. If <code>n</code> is
+ * negative, no bytes are skipped.
+ *
+ * @param n the number of bytes to be skipped.
+ * @return the actual number of bytes skipped.
+ * @exception IOException if an I/O error occurs.
+ * ChecksumException if the chunk to skip to is corrupted
+ */
+ public synchronized long skip(long n) throws IOException {
+ long curPos = getPos();
+ long fileLength = getFileLength();
+ if( n+curPos > fileLength ) {
+ n = fileLength - curPos;
+ }
+ return super.skip(n);
+ }
+
+ /**
+ * Seek to the given position in the stream.
+ * The next read() will be from that position.
+ *
+ * <p>This method does not allow seek past the end of the file.
+ * This produces IOException.
+ *
+ * @param pos the postion to seek to.
+ * @exception IOException if an I/O error occurs or seeks after EOF
+ * ChecksumException if the chunk to seek to is corrupted
+ */
+
+ public synchronized void seek(long pos) throws IOException {
+ if(pos>getFileLength()) {
+ throw new IOException("Cannot seek after EOF");
+ }
+ super.seek(pos);
+ }
+
}
/**
Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java Mon Jul 23 12:00:23 2007
@@ -297,7 +297,23 @@
return count-pos;
}
- @Override
+ /**
+ * Skips over and discards <code>n</code> bytes of data from the
+ * input stream.
+ *
+ * <p>This method may skip more bytes than are remaining in the backing
+ * file. This produces no exception and the number of bytes skipped
+ * may include some number of bytes that were beyond the EOF of the
+ * backing file. Attempting to read from the stream after skipping past
+ * the end will result in -1 indicating the end of the file.
+ *
+ *<p>If <code>n</code> is negative, no bytes are skipped.
+ *
+ * @param n the number of bytes to be skipped.
+ * @return the actual number of bytes skipped.
+ * @exception IOException if an I/O error occurs.
+ * ChecksumException if the chunk to skip to is corrupted
+ */
public synchronized long skip(long n) throws IOException {
if (n <= 0) {
return 0;
@@ -307,7 +323,19 @@
return n;
}
- @Override
+ /**
+ * Seek to the given position in the stream.
+ * The next read() will be from that position.
+ *
+ * <p>This method may seek past the end of the file.
+ * This produces no exception and an attempt to read from
+ * the stream will result in -1 indicating the end of the file.
+ *
+ * @param pos the postion to seek to.
+ * @exception IOException if an I/O error occurs.
+ * ChecksumException if the chunk to seek to is corrupted
+ */
+
public synchronized void seek(long pos) throws IOException {
if( pos<0 ) {
return;
@@ -328,10 +356,7 @@
// scan to the desired position
int delta = (int)(pos - chunkPos);
if( delta > 0) {
- int nread = readFully(this, new byte[delta], 0, delta);
- if (nread < delta) {
- throw new IOException("Cannot seek after EOF");
- }
+ readFully(this, new byte[delta], 0, delta);
}
}
Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java Mon Jul 23 12:00:23 2007
@@ -181,6 +181,12 @@
stm.seek(0);
assertEquals(stm.skip(FILE_SIZE), FILE_SIZE);
+ assertEquals(stm.skip(10), 0);
+
+ stm.seek(0);
+ assertEquals(stm.skip(FILE_SIZE+10), FILE_SIZE);
+ stm.seek(10);
+ assertEquals(stm.skip(FILE_SIZE), FILE_SIZE-10);
}
private void cleanupFile(FileSystem fileSys, Path name) throws IOException {