You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jz...@apache.org on 2017/08/24 07:50:35 UTC
hadoop git commit: HDFS-12318. Fix IOException condition for openInfo
in DFSInputStream. Contributed by legend.
Repository: hadoop
Updated Branches:
refs/heads/trunk 652dd434d -> 3123895db
HDFS-12318. Fix IOException condition for openInfo in DFSInputStream. Contributed by legend.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3123895d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3123895d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3123895d
Branch: refs/heads/trunk
Commit: 3123895db0518552eb357093fddf0e8ff1a6cadf
Parents: 652dd43
Author: John Zhuge <jz...@apache.org>
Authored: Thu Aug 24 00:26:37 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Thu Aug 24 00:26:37 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +++-
.../apache/hadoop/hdfs/TestDFSInputStream.java | 41 ++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 97d3de4..e3d7ade 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -182,6 +182,11 @@ public class DFSInputStream extends FSInputStream
openInfo(false);
}
+ @VisibleForTesting
+ public long getlastBlockBeingWrittenLengthForTesting() {
+ return lastBlockBeingWrittenLength;
+ }
+
/**
* Grab the open-file info from namenode
* @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -209,7 +214,8 @@ public class DFSInputStream extends FSInputStream
}
retriesForLastBlockLength--;
}
- if (retriesForLastBlockLength == 0) {
+ if (lastBlockBeingWrittenLength == -1
+ && retriesForLastBlockLength == 0) {
throw new IOException("Could not obtain the last block locations.");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3123895d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
index 2012258..eb4f124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+
import org.junit.Assume;
import org.junit.Test;
@@ -135,4 +138,42 @@ public class TestDFSInputStream {
cluster.shutdown();
}
}
+
+ @Test(timeout=60000)
+ public void testOpenInfo() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setInt(Retry.TIMES_GET_LAST_BLOCK_LENGTH_KEY, 0);
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ try {
+ DistributedFileSystem fs = cluster.getFileSystem();
+
+ int chunkSize = 512;
+ Random r = new Random(12345L);
+ byte[] data = new byte[chunkSize];
+ r.nextBytes(data);
+
+ Path file = new Path("/testfile");
+ try(FSDataOutputStream fout = fs.create(file)) {
+ fout.write(data);
+ }
+
+ DfsClientConf dcconf = new DfsClientConf(conf);
+ int retryTimesForGetLastBlockLength =
+ dcconf.getRetryTimesForGetLastBlockLength();
+ assertEquals(0, retryTimesForGetLastBlockLength);
+
+ try(DFSInputStream fin = fs.dfs.open("/testfile")) {
+ long flen = fin.getFileLength();
+ assertEquals(chunkSize, flen);
+
+ long lastBlockBeingWrittenLength =
+ fin.getlastBlockBeingWrittenLengthForTesting();
+ assertEquals(0, lastBlockBeingWrittenLength);
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org