You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2015/03/04 03:11:23 UTC
[1/2] hadoop git commit: HDFS-7682.
{{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes
non-snapshotted content. Contributed by Charles Lamb.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 65bfde552 -> bce3d442f
refs/heads/trunk e2262d3d1 -> f2d7a67a2
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes non-snapshotted content. Contributed by Charles Lamb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2d7a67a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2d7a67a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2d7a67a
Branch: refs/heads/trunk
Commit: f2d7a67a2c1d9dde10ed3171fdec65dff885afcc
Parents: e2262d3
Author: Aaron T. Myers <at...@apache.org>
Authored: Tue Mar 3 18:08:59 2015 -0800
Committer: Aaron T. Myers <at...@apache.org>
Committed: Tue Mar 3 18:08:59 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +++
.../snapshot/TestSnapshotFileLength.java | 25 +++++++++++++++++---
3 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4e7b919..7ff3c78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1080,6 +1080,9 @@ Release 2.7.0 - UNRELEASED
HDFS-6565. Use jackson instead jetty json in hdfs-client.
(Akira Ajisaka via wheat9)
+ HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file
+ includes non-snapshotted content. (Charles Lamb via atm)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index abcd847..aac7b51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2220,6 +2220,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
// get block checksum for each block
long remaining = length;
+ if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+ remaining = Math.min(length, blockLocations.getFileLength());
+ }
for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
if (refetchBlocks) { // refetch to get fresh tokens
blockLocations = callGetBlockLocations(namenode, src, 0, length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2d7a67a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 98aafc1..d53140f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
-
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -29,8 +29,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -103,17 +104,35 @@ public class TestSnapshotFileLength {
Path file1snap1
= SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name);
+ final FileChecksum snapChksum1 = hdfs.getFileChecksum(file1snap1);
+ assertThat("file and snapshot file checksums are not equal",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+
// Append to the file.
FSDataOutputStream out = hdfs.append(file1);
+ // Nothing has been appended yet. All checksums should still be equal.
+ assertThat("file and snapshot checksums (open for append) are not equal",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+ assertThat("snapshot checksum (post-open for append) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
try {
AppendTestUtil.write(out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1);
assertThat("Wrong data size in snapshot.",
dataFromSnapshot.length, is(origLen));
+ // Verify that checksum didn't change
+ assertThat("snapshot file checksum (pre-close) has changed",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+ assertThat("snapshot checksum (post-append) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
} finally {
out.close();
}
+ assertThat("file and snapshot file checksums (post-close) are equal",
+ hdfs.getFileChecksum(file1), not(snapChksum1));
+ assertThat("snapshot file checksum (post-close) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.getFileStatus(file1);
[2/2] hadoop git commit: HDFS-7682.
{{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes
non-snapshotted content. Contributed by Charles Lamb. (cherry picked from
commit f2d7a67a2c1d9dde10ed3171fdec65dff885afcc)
Posted by at...@apache.org.
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes non-snapshotted content. Contributed by Charles Lamb.
(cherry picked from commit f2d7a67a2c1d9dde10ed3171fdec65dff885afcc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bce3d442
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bce3d442
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bce3d442
Branch: refs/heads/branch-2
Commit: bce3d442ff08ee1e730b0bac112439d6a6931917
Parents: 65bfde5
Author: Aaron T. Myers <at...@apache.org>
Authored: Tue Mar 3 18:08:59 2015 -0800
Committer: Aaron T. Myers <at...@apache.org>
Committed: Tue Mar 3 18:09:31 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +++
.../snapshot/TestSnapshotFileLength.java | 25 +++++++++++++++++---
3 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 69a410f..bff45bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -774,6 +774,9 @@ Release 2.7.0 - UNRELEASED
HDFS-6565. Use jackson instead jetty json in hdfs-client.
(Akira Ajisaka via wheat9)
+ HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file
+ includes non-snapshotted content. (Charles Lamb via atm)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 6f96126..ba6a1d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2219,6 +2219,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
// get block checksum for each block
long remaining = length;
+ if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+ remaining = Math.min(length, blockLocations.getFileLength());
+ }
for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
if (refetchBlocks) { // refetch to get fresh tokens
blockLocations = callGetBlockLocations(namenode, src, 0, length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 98aafc1..d53140f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
-
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -29,8 +29,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -103,17 +104,35 @@ public class TestSnapshotFileLength {
Path file1snap1
= SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name);
+ final FileChecksum snapChksum1 = hdfs.getFileChecksum(file1snap1);
+ assertThat("file and snapshot file checksums are not equal",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+
// Append to the file.
FSDataOutputStream out = hdfs.append(file1);
+ // Nothing has been appended yet. All checksums should still be equal.
+ assertThat("file and snapshot checksums (open for append) are not equal",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+ assertThat("snapshot checksum (post-open for append) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
try {
AppendTestUtil.write(out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1);
assertThat("Wrong data size in snapshot.",
dataFromSnapshot.length, is(origLen));
+ // Verify that checksum didn't change
+ assertThat("snapshot file checksum (pre-close) has changed",
+ hdfs.getFileChecksum(file1), is(snapChksum1));
+ assertThat("snapshot checksum (post-append) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
} finally {
out.close();
}
+ assertThat("file and snapshot file checksums (post-close) are equal",
+ hdfs.getFileChecksum(file1), not(snapChksum1));
+ assertThat("snapshot file checksum (post-close) has changed",
+ hdfs.getFileChecksum(file1snap1), is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.getFileStatus(file1);