You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/06/20 23:25:44 UTC
[39/50] [abbrv] hadoop git commit: HDFS-8626. Reserved RBW space is
not released if creation of RBW File fails. (Contributed by kanaka kumar
avvaru)
HDFS-8626. Reserved RBW space is not released if creation of RBW File fails. (Contributed by kanaka kumar avvaru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55278c01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55278c01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55278c01
Branch: refs/heads/HDFS-7240
Commit: 55278c012fbd5dcabee0276f8f42d0ab920cec93
Parents: 49f5d20
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Jun 19 18:23:34 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Jun 19 18:23:34 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../datanode/fsdataset/impl/FsVolumeImpl.java | 7 ++-
.../fsdataset/impl/TestRbwSpaceReservation.java | 45 ++++++++++++++++++++
3 files changed, 54 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55278c01/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 749a3ce..a93a214 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1064,6 +1064,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8633. Fix setting of dfs.datanode.readahead.bytes in hdfs-default.xml
to match DFSConfigKeys. (Ray Chiang via Yongjun Zhang)
+ HDFS-8626. Reserved RBW space is not released if creation of RBW File
+ fails. (kanaka kumar avvaru via Arpit Agarwal)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55278c01/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 49a56bb..e90f5d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -780,7 +780,12 @@ public class FsVolumeImpl implements FsVolumeSpi {
File createRbwFile(String bpid, Block b) throws IOException {
checkReference();
reserveSpaceForRbw(b.getNumBytes());
- return getBlockPoolSlice(bpid).createRbwFile(b);
+ try {
+ return getBlockPoolSlice(bpid).createRbwFile(b);
+ } catch (IOException exception) {
+ releaseReservedSpace(b.getNumBytes());
+ throw exception;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55278c01/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java
index 9b83b39..44bf81b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java
@@ -27,10 +27,13 @@ import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.*;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
@@ -42,9 +45,12 @@ import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeoutException;
@@ -279,6 +285,45 @@ public class TestRbwSpaceReservation {
}
}
+ @SuppressWarnings("unchecked")
+ @Test(timeout = 30000)
+ public void testRBWFileCreationError() throws Exception {
+
+ final short replication = 1;
+ startCluster(BLOCK_SIZE, replication, -1);
+
+ final FsVolumeImpl fsVolumeImpl = (FsVolumeImpl) cluster.getDataNodes()
+ .get(0).getFSDataset().getFsVolumeReferences().get(0);
+ final String methodName = GenericTestUtils.getMethodName();
+ final Path file = new Path("/" + methodName + ".01.dat");
+
+ // Mock BlockPoolSlice so that RBW file creation gives IOExcception
+ BlockPoolSlice blockPoolSlice = Mockito.mock(BlockPoolSlice.class);
+ Mockito.when(blockPoolSlice.createRbwFile((Block) Mockito.any()))
+ .thenThrow(new IOException("Synthetic IO Exception Throgh MOCK"));
+
+ Field field = FsVolumeImpl.class.getDeclaredField("bpSlices");
+ field.setAccessible(true);
+ Map<String, BlockPoolSlice> bpSlices = (Map<String, BlockPoolSlice>) field
+ .get(fsVolumeImpl);
+ bpSlices.put(fsVolumeImpl.getBlockPoolList()[0], blockPoolSlice);
+
+ try {
+ // Write 1 byte to the file
+ FSDataOutputStream os = fs.create(file, replication);
+ os.write(new byte[1]);
+ os.hsync();
+ os.close();
+ fail("Expecting IOException file creation failure");
+ } catch (IOException e) {
+ // Exception can be ignored (expected)
+ }
+
+ // Ensure RBW space reserved is released
+ assertTrue("Expected ZERO but got " + fsVolumeImpl.getReservedForRbw(),
+ fsVolumeImpl.getReservedForRbw() == 0);
+ }
+
/**
* Stress test to ensure we are not leaking reserved space.
* @throws IOException