You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/05/14 12:25:44 UTC
[2/2] hadoop git commit: HDFS-8150. Make getFileChecksum fail for
blocks under construction (Contributed by J.Andreina)
HDFS-8150. Make getFileChecksum fail for blocks under construction (Contributed by J.Andreina)
(cherry picked from commit def9136e0259e118e6fd7b656260765d28ac9ae6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bbcffa5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bbcffa5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bbcffa5
Branch: refs/heads/branch-2
Commit: 4bbcffa510f7f93822c68204c408a1acebf431bc
Parents: 085841d
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu May 14 15:54:51 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu May 14 15:55:20 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 8 ++++++++
.../apache/hadoop/hdfs/TestGetFileChecksum.java | 19 +++++++++++++++++++
.../snapshot/TestSnapshotFileLength.java | 17 +++++++++++++----
4 files changed, 43 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 17ee336..5961d34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -428,6 +428,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7728. Avoid updating quota usage while loading edits.
(Jing Zhao via wheat9)
+ HDFS-8150. Make getFileChecksum fail for blocks under construction
+ (J.Andreina via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 216c843..60c53b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1871,6 +1871,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if (null == blockLocations) {
throw new FileNotFoundException("File does not exist: " + src);
}
+ if (blockLocations.isUnderConstruction()) {
+ throw new IOException("Fail to get checksum, since file " + src
+ + " is under construction.");
+ }
List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
final DataOutputBuffer md5out = new DataOutputBuffer();
int bytesPerCRC = -1;
@@ -1890,6 +1894,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if (null == blockLocations) {
throw new FileNotFoundException("File does not exist: " + src);
}
+ if (blockLocations.isUnderConstruction()) {
+ throw new IOException("Fail to get checksum, since file " + src
+ + " is under construction.");
+ }
locatedblocks = blockLocations.getLocatedBlocks();
refetchBlocks = false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
index 0e56ba7..814261f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
@@ -17,7 +17,12 @@
*/
package org.apache.hadoop.hdfs;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.Path;
import org.junit.After;
@@ -68,6 +73,20 @@ public class TestGetFileChecksum {
}
@Test
+ public void testGetFileChecksumForBlocksUnderConstruction() {
+ try {
+ FSDataOutputStream file = dfs.create(new Path("/testFile"));
+ file.write("Performance Testing".getBytes());
+ dfs.getFileChecksum(new Path("/testFile"));
+ fail("getFileChecksum should fail for files "
+ + "with blocks under construction");
+ } catch (IOException ie) {
+ Assert.assertTrue(ie.getMessage().contains(
+ "Fail to get checksum, since file /testFile "
+ + "is under construction."));
+ }
+ }
+ @Test
public void testGetFileChecksum() throws Exception {
testGetFileChecksum(new Path("/foo"), BLOCKSIZE / 4);
testGetFileChecksum(new Path("/bar"), BLOCKSIZE / 4 - 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bbcffa5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index d53140f..90e499d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.io.PrintStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,6 +33,8 @@ import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -111,8 +114,16 @@ public class TestSnapshotFileLength {
// Append to the file.
FSDataOutputStream out = hdfs.append(file1);
// Nothing has been appended yet. All checksums should still be equal.
- assertThat("file and snapshot checksums (open for append) are not equal",
- hdfs.getFileChecksum(file1), is(snapChksum1));
+ // HDFS-8150:Fetching checksum for file under construction should fail
+ try {
+ hdfs.getFileChecksum(file1);
+ fail("getFileChecksum should fail for files "
+ + "with blocks under construction");
+ } catch (IOException ie) {
+ assertTrue(ie.getMessage().contains(
+ "Fail to get checksum, since file " + file1
+ + " is under construction."));
+ }
assertThat("snapshot checksum (post-open for append) has changed",
hdfs.getFileChecksum(file1snap1), is(snapChksum1));
try {
@@ -122,8 +133,6 @@ public class TestSnapshotFileLength {
assertThat("Wrong data size in snapshot.",
dataFromSnapshot.length, is(origLen));
// Verify that checksum didn't change
- assertThat("snapshot file checksum (pre-close) has changed",
- hdfs.getFileChecksum(file1), is(snapChksum1));
assertThat("snapshot checksum (post-append) has changed",
hdfs.getFileChecksum(file1snap1), is(snapChksum1));
} finally {