You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/10/31 18:23:52 UTC
[41/50] [abbrv] hadoop git commit: HDDS-754. VolumeInfo#getScmUsed
throws NPE. Contributed by Hanisha Koneru.
HDDS-754. VolumeInfo#getScmUsed throws NPE.
Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773f0d15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773f0d15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773f0d15
Branch: refs/heads/HDFS-12943
Commit: 773f0d1519715e3ddf77c139998cc12d7447da66
Parents: e33b61f
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 30 19:17:57 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Oct 30 19:17:57 2018 -0700
----------------------------------------------------------------------
.../container/common/volume/VolumeInfo.java | 19 +++++++++++++++++--
.../ozone/container/common/volume/VolumeSet.java | 11 +++++++----
.../container/common/volume/TestHddsVolume.java | 9 ++++++---
.../container/common/volume/TestVolumeSet.java | 4 +++-
.../hdds/scm/pipeline/TestNodeFailure.java | 3 ++-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 8 ++++++++
.../hadoop/ozone/MiniOzoneClusterImpl.java | 16 +++++++++++++++-
7 files changed, 58 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 62fca63..0de9f18 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -95,15 +95,30 @@ public class VolumeInfo {
this.usage = new VolumeUsage(root, b.conf);
}
- public long getCapacity() {
- return configuredCapacity < 0 ? usage.getCapacity() : configuredCapacity;
+ public long getCapacity() throws IOException {
+ if (configuredCapacity < 0) {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error" +
+ " is usually seen during DataNode shutdown.");
+ }
+ return usage.getCapacity();
+ }
+ return configuredCapacity;
}
public long getAvailable() throws IOException {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error " +
+ "is usually seen during DataNode shutdown.");
+ }
return usage.getAvailable();
}
public long getScmUsed() throws IOException {
+ if (usage == null) {
+ throw new IOException("Volume Usage thread is not running. This error " +
+ "is usually seen during DataNode shutdown.");
+ }
return usage.getScmUsed();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
index 5b6b823..d30dd89 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
@@ -372,18 +372,21 @@ public class VolumeSet {
for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) {
hddsVolume = entry.getValue();
VolumeInfo volumeInfo = hddsVolume.getVolumeInfo();
- long scmUsed = 0;
- long remaining = 0;
+ long scmUsed;
+ long remaining;
+ long capacity;
failed = false;
try {
scmUsed = volumeInfo.getScmUsed();
remaining = volumeInfo.getAvailable();
+ capacity = volumeInfo.getCapacity();
} catch (IOException ex) {
LOG.warn("Failed to get scmUsed and remaining for container " +
- "storage location {}", volumeInfo.getRootDir());
+ "storage location {}", volumeInfo.getRootDir(), ex);
// reset scmUsed and remaining if df/du failed.
scmUsed = 0;
remaining = 0;
+ capacity = 0;
failed = true;
}
@@ -392,7 +395,7 @@ public class VolumeSet {
builder.setStorageLocation(volumeInfo.getRootDir())
.setId(hddsVolume.getStorageID())
.setFailed(failed)
- .setCapacity(hddsVolume.getCapacity())
+ .setCapacity(capacity)
.setRemaining(remaining)
.setScmUsed(scmUsed)
.setStorageType(hddsVolume.getStorageType());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 7755345..6b46762 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -31,6 +31,7 @@ import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import java.io.File;
+import java.io.IOException;
import java.util.Properties;
import java.util.UUID;
@@ -134,12 +135,14 @@ public class TestHddsVolume {
scmUsedFile.exists());
try {
- // Volume.getAvailable() should fail with NullPointerException as usage
- // is shutdown.
+ // Volume.getAvailable() should fail with IOException
+ // as usage thread is shutdown.
volume.getAvailable();
fail("HddsVolume#shutdown test failed");
} catch (Exception ex){
- assertTrue(ex instanceof NullPointerException);
+ assertTrue(ex instanceof IOException);
+ assertTrue(ex.getMessage().contains(
+ "Volume Usage thread is not running."));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
index fca68b1..7bb8a43 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
@@ -222,8 +222,10 @@ public class TestVolumeSet {
// getAvailable() should throw null pointer exception as usage is null.
volume.getAvailable();
fail("Volume shutdown failed.");
- } catch (NullPointerException ex) {
+ } catch (IOException ex) {
// Do Nothing. Exception is expected.
+ assertTrue(ex.getMessage().contains(
+ "Volume Usage thread is not running."));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
index 9a1c705..618cd8e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
@@ -99,7 +99,7 @@ public class TestNodeFailure {
}
}
- @Test
+ @Test(timeout = 300_000L)
public void testPipelineFail() throws InterruptedException, IOException,
TimeoutException {
Assert.assertEquals(ratisContainer1.getPipeline().getPipelineState(),
@@ -118,6 +118,7 @@ public class TestNodeFailure {
pipelineManager.getPipeline(ratisContainer2.getPipeline().getId())
.getPipelineState());
// Now restart the datanode and make sure that a new pipeline is created.
+ cluster.setWaitForClusterToBeReadyTimeout(300000);
cluster.restartHddsDatanode(dnToFail, true);
ContainerWithPipeline ratisContainer3 =
containerManager.allocateContainer(RATIS, THREE, "testOwner");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3aad7f7..15bf8d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -67,6 +67,14 @@ public interface MiniOzoneCluster {
void waitForClusterToBeReady() throws TimeoutException, InterruptedException;
/**
+ * Sets the timeout value after which
+ * {@link MiniOzoneCluster#waitForClusterToBeReady} times out.
+ *
+ * @param timeoutInMs timeout value in milliseconds
+ */
+ void setWaitForClusterToBeReadyTimeout(int timeoutInMs);
+
+ /**
* Waits/blocks till the cluster is out of chill mode.
*
* @throws TimeoutException TimeoutException In case of timeout
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 11bc0e0..6c0f408 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -90,6 +90,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
private final OzoneManager ozoneManager;
private final List<HddsDatanodeService> hddsDatanodes;
+ // Timeout for the cluster to be ready
+ private int waitForClusterToBeReadyTimeout = 60000; // 1 min
+
/**
* Creates a new MiniOzoneCluster.
*
@@ -122,7 +125,18 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
isReady? "Cluster is ready" : "Waiting for cluster to be ready",
healthy, hddsDatanodes.size());
return isReady;
- }, 1000, 60 * 1000); //wait for 1 min.
+ }, 1000, waitForClusterToBeReadyTimeout);
+ }
+
+ /**
+ * Sets the timeout value after which
+ * {@link MiniOzoneClusterImpl#waitForClusterToBeReady} times out.
+ *
+ * @param timeoutInMs timeout value in milliseconds
+ */
+ @Override
+ public void setWaitForClusterToBeReadyTimeout(int timeoutInMs) {
+ waitForClusterToBeReadyTimeout = timeoutInMs;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org