You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/05/30 03:26:08 UTC
[hadoop] branch branch-2.9 updated: HDFS-14514. Actual read size of
open file in encryption zone still larger than listing size even after
enabling HDFS-11402 in Hadoop 2. Contributed by Siyao Meng,
Stephen O'Donnell.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new ebd2c77 HDFS-14514. Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2. Contributed by Siyao Meng, Stephen O'Donnell.
ebd2c77 is described below
commit ebd2c77da6564d99a2afa601bc5e9c859e8140a9
Author: Siyao Meng <sm...@cloudera.com>
AuthorDate: Wed May 29 17:27:39 2019 -0700
HDFS-14514. Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2. Contributed by Siyao Meng, Stephen O'Donnell.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
Co-authored-by: Stephen O'Donnell <so...@cloudera.com>
(cherry picked from commit cecba551aa3b03142830cfd58f548cb6b125ee7b)
---
.../org/apache/hadoop/hdfs/DFSInputStream.java | 30 ++++++++--------------
.../apache/hadoop/hdfs/TestEncryptionZones.java | 12 +++++++--
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index cff259f..db42a8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -844,26 +844,18 @@ public class DFSInputStream extends FSInputStream
@Override
public int doRead(BlockReader blockReader, int off, int len)
throws IOException {
- int oldpos = buf.position();
- int oldlimit = buf.limit();
- boolean success = false;
- try {
- int ret = blockReader.read(buf);
- success = true;
- updateReadStatistics(readStatistics, ret, blockReader);
- dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
- ret);
- if (ret == 0) {
- DFSClient.LOG.warn("zero");
- }
- return ret;
- } finally {
- if (!success) {
- // Reset to original state so that retries work correctly.
- buf.position(oldpos);
- buf.limit(oldlimit);
- }
+ ByteBuffer tmpBuf = buf.duplicate();
+ tmpBuf.limit(tmpBuf.position() + len);
+ int nRead = blockReader.read(tmpBuf);
+ updateReadStatistics(readStatistics, nRead, blockReader);
+ dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+ nRead);
+ if (nRead == 0) {
+ DFSClient.LOG.warn("zero");
+ } else if (nRead > 0) {
+ buf.position(buf.position() + nRead);
}
+ return nRead;
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 4c92af5..37fad8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -184,6 +184,8 @@ public class TestEncryptionZones {
testRootDir = new File(testRoot).getAbsoluteFile();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
getKeyProviderURI());
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES,
+ true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
@@ -1389,6 +1391,14 @@ public class TestEncryptionZones {
assertEquals("Got unexpected ez path", zone.toString(),
dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
+ // Append the file
+ DFSTestUtil.appendFile(fs, zoneFile, len);
+ // Verify file content in the snapshot
+ final Path snapshottedZoneFile = new Path(
+ snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
+ assertEquals("Contents of snapshotted file have changed unexpectedly",
+ contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
+
// Now delete the encryption zone, recreate the dir, and take another
// snapshot
fsWrapper.delete(zone, true);
@@ -1441,8 +1451,6 @@ public class TestEncryptionZones {
assertEquals("Unexpected ez key", TEST_KEY2, listZone.getKeyName());
// Verify contents of the snapshotted file
- final Path snapshottedZoneFile = new Path(
- snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
assertEquals("Contents of snapshotted file have changed unexpectedly",
contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org