You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2020/03/26 00:00:53 UTC
[hadoop] branch branch-3.1 updated (9c6dd8c -> d64f688)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a change to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 9c6dd8c YARN-10200. Add number of containers to RMAppManager summary
new f1a19b7 HDFS-14986. ReplicaCachingGetSpaceUsed throws ConcurrentModificationException. Contributed by Aiphago.
new 4891f24 HDFS-15158. The number of failed volumes mismatch with volumeFailures of Datanode metrics. Contributed by Yang Yun.
new d64f688 HDFS-15219. DFS Client will stuck when ResponseProcessor.run throw Error (#1902). Contributed by zhengchenyu.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/hadoop/fs/CachingGetSpaceUsed.java | 34 +++++++++++--
.../java/org/apache/hadoop/hdfs/DataStreamer.java | 2 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 12 ++---
.../server/datanode/fsdataset/FsDatasetSpi.java | 6 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 ++---
.../fsdataset/impl/ReplicaCachingGetSpaceUsed.java | 1 +
.../server/datanode/metrics/DataNodeMetrics.java | 6 +--
.../server/datanode/TestDataNodeVolumeFailure.java | 26 ++++++++++
.../impl/TestReplicaCachingGetSpaceUsed.java | 55 ++++++++++++++++++++++
9 files changed, 134 insertions(+), 20 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDFS-14986. ReplicaCachingGetSpaceUsed throws
ConcurrentModificationException. Contributed by Aiphago.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f1a19b7a3f590f8b1876deb1bae6dfb4bf840edb
Author: Yiqun Lin <yq...@apache.org>
AuthorDate: Thu Nov 28 10:43:35 2019 +0800
HDFS-14986. ReplicaCachingGetSpaceUsed throws ConcurrentModificationException. Contributed by Aiphago.
(cherry picked from commit 2b452b4e6063072b2bec491edd3f412eb7ac21f3)
---
.../org/apache/hadoop/fs/CachingGetSpaceUsed.java | 34 +++++++++++--
.../server/datanode/fsdataset/FsDatasetSpi.java | 6 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 12 ++---
.../fsdataset/impl/ReplicaCachingGetSpaceUsed.java | 1 +
.../impl/TestReplicaCachingGetSpaceUsed.java | 55 ++++++++++++++++++++++
5 files changed, 98 insertions(+), 10 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
index 92476d7..58dc82d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
@@ -47,6 +47,7 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
private final long jitter;
private final String dirPath;
private Thread refreshUsed;
+ private boolean shouldFirstRefresh;
/**
* This is the constructor used by the builder.
@@ -79,16 +80,30 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
this.refreshInterval = interval;
this.jitter = jitter;
this.used.set(initialUsed);
+ this.shouldFirstRefresh = true;
}
void init() {
if (used.get() < 0) {
used.set(0);
+ if (!shouldFirstRefresh) {
+ // Skip initial refresh operation, so we need to do first refresh
+ // operation immediately in refresh thread.
+ initRefeshThread(true);
+ return;
+ }
refresh();
}
+ initRefeshThread(false);
+ }
+ /**
+ * RunImmediately should set true, if we skip the first refresh.
+ * @param runImmediately The param default should be false.
+ */
+ private void initRefeshThread (boolean runImmediately) {
if (refreshInterval > 0) {
- refreshUsed = new Thread(new RefreshThread(this),
+ refreshUsed = new Thread(new RefreshThread(this, runImmediately),
"refreshUsed-" + dirPath);
refreshUsed.setDaemon(true);
refreshUsed.start();
@@ -101,6 +116,14 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
protected abstract void refresh();
/**
+ * Reset that if we need to do the first refresh.
+ * @param shouldFirstRefresh The flag value to set.
+ */
+ protected void setShouldFirstRefresh(boolean shouldFirstRefresh) {
+ this.shouldFirstRefresh = shouldFirstRefresh;
+ }
+
+ /**
* @return an estimate of space used in the directory path.
*/
@Override public long getUsed() throws IOException {
@@ -156,9 +179,11 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
private static final class RefreshThread implements Runnable {
final CachingGetSpaceUsed spaceUsed;
+ private boolean runImmediately;
- RefreshThread(CachingGetSpaceUsed spaceUsed) {
+ RefreshThread(CachingGetSpaceUsed spaceUsed, boolean runImmediately) {
this.spaceUsed = spaceUsed;
+ this.runImmediately = runImmediately;
}
@Override
@@ -176,7 +201,10 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
}
// Make sure that after the jitter we didn't end up at 0.
refreshInterval = Math.max(refreshInterval, 1);
- Thread.sleep(refreshInterval);
+ if (!runImmediately) {
+ Thread.sleep(refreshInterval);
+ }
+ runImmediately = false;
// update the used variable
spaceUsed.refresh();
} catch (InterruptedException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 78a5cfc..578c390 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -661,5 +661,11 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
*/
AutoCloseableLock acquireDatasetLock();
+ /**
+ * Deep copy the replica info belonging to given block pool.
+ * @param bpid Specified block pool id.
+ * @return A set of replica info.
+ * @throws IOException
+ */
Set<? extends Replica> deepCopyReplica(String bpid) throws IOException;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index ac92ae4..957b306 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -198,16 +198,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
- /**
- * The deepCopyReplica call doesn't use the datasetock since it will lead the
- * potential deadlock with the {@link FsVolumeList#addBlockPool} call.
- */
@Override
public Set<? extends Replica> deepCopyReplica(String bpid)
throws IOException {
- Set<? extends Replica> replicas =
- new HashSet<>(volumeMap.replicas(bpid) == null ? Collections.EMPTY_SET
- : volumeMap.replicas(bpid));
+ Set<? extends Replica> replicas = null;
+ try (AutoCloseableLock lock = datasetLock.acquire()) {
+ replicas = new HashSet<>(volumeMap.replicas(bpid) == null ? Collections.
+ EMPTY_SET : volumeMap.replicas(bpid));
+ }
return Collections.unmodifiableSet(replicas);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaCachingGetSpaceUsed.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaCachingGetSpaceUsed.java
index 2c1c16e..5acc3c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaCachingGetSpaceUsed.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaCachingGetSpaceUsed.java
@@ -59,6 +59,7 @@ public class ReplicaCachingGetSpaceUsed extends FSCachingGetSpaceUsed {
public ReplicaCachingGetSpaceUsed(Builder builder) throws IOException {
super(builder);
+ setShouldFirstRefresh(false);
volume = builder.getVolume();
bpid = builder.getBpid();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaCachingGetSpaceUsed.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaCachingGetSpaceUsed.java
index 45a3916..6abf523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaCachingGetSpaceUsed.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaCachingGetSpaceUsed.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+import org.apache.commons.lang.math.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CachingGetSpaceUsed;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -27,8 +28,11 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.Replica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -36,6 +40,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
+import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY;
import static org.junit.Assert.assertEquals;
@@ -145,4 +150,54 @@ public class TestReplicaCachingGetSpaceUsed {
fs.delete(new Path("/testReplicaCachingGetSpaceUsedByRBWReplica"), true);
}
+
+ @Test(timeout = 15000)
+ public void testFsDatasetImplDeepCopyReplica() {
+ FsDatasetSpi<?> fsDataset = dataNode.getFSDataset();
+ ModifyThread modifyThread = new ModifyThread();
+ modifyThread.start();
+ String bpid = cluster.getNamesystem(0).getBlockPoolId();
+ int retryTimes = 10;
+
+ while (retryTimes > 0) {
+ try {
+ Set<? extends Replica> replicas = fsDataset.deepCopyReplica(bpid);
+ if (replicas.size() > 0) {
+ retryTimes--;
+ }
+ } catch (IOException e) {
+ modifyThread.setShouldRun(false);
+ Assert.fail("Encounter IOException when deep copy replica.");
+ }
+ }
+ modifyThread.setShouldRun(false);
+ }
+
+ private class ModifyThread extends Thread {
+ private boolean shouldRun = true;
+
+ @Override
+ public void run() {
+ FSDataOutputStream os = null;
+ while (shouldRun) {
+ try {
+ int id = RandomUtils.nextInt();
+ os = fs.create(new Path("/testFsDatasetImplDeepCopyReplica/" + id));
+ byte[] bytes = new byte[2048];
+ InputStream is = new ByteArrayInputStream(bytes);
+ IOUtils.copyBytes(is, os, bytes.length);
+ os.hsync();
+ os.close();
+ } catch (IOException e) {}
+ }
+
+ try {
+ fs.delete(new Path("/testFsDatasetImplDeepCopyReplica"), true);
+ } catch (IOException e) {}
+ }
+
+ private void setShouldRun(boolean shouldRun) {
+ this.shouldRun = shouldRun;
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDFS-15158. The number of failed volumes mismatch
with volumeFailures of Datanode metrics. Contributed by Yang Yun.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4891f24a94a7a2a293dc379d23e93907bbb40526
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Sun Feb 9 23:19:40 2020 +0530
HDFS-15158. The number of failed volumes mismatch with volumeFailures of Datanode metrics. Contributed by Yang Yun.
(cherry picked from commit 6191d4b4a0919863fda78e549ab6c60022e3ebc2)
(cherry picked from commit 1e3b0df6abcc1252907c41aaedb3e7e257bce497)
---
.../hadoop/hdfs/server/datanode/DataNode.java | 12 +++++-----
.../server/datanode/metrics/DataNodeMetrics.java | 6 ++---
.../server/datanode/TestDataNodeVolumeFailure.java | 26 ++++++++++++++++++++++
3 files changed, 35 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 4cb2d93..62e4262 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2192,7 +2192,7 @@ public class DataNode extends ReconfigurableBase
});
}
- private void handleDiskError(String failedVolumes) {
+ private void handleDiskError(String failedVolumes, int failedNumber) {
final boolean hasEnoughResources = data.hasEnoughResource();
LOG.warn("DataNode.handleDiskError on: " +
"[{}] Keep Running: {}", failedVolumes, hasEnoughResources);
@@ -2201,7 +2201,7 @@ public class DataNode extends ReconfigurableBase
// shutdown the DN completely.
int dpError = hasEnoughResources ? DatanodeProtocol.DISK_ERROR
: DatanodeProtocol.FATAL_DISK_ERROR;
- metrics.incrVolumeFailures();
+ metrics.incrVolumeFailures(failedNumber);
//inform NameNodes
for(BPOfferService bpos: blockPoolManager.getAllNamenodeThreads()) {
@@ -3408,8 +3408,8 @@ public class DataNode extends ReconfigurableBase
}
data.handleVolumeFailures(unhealthyVolumes);
- Set<StorageLocation> unhealthyLocations = new HashSet<>(
- unhealthyVolumes.size());
+ int failedNumber = unhealthyVolumes.size();
+ Set<StorageLocation> unhealthyLocations = new HashSet<>(failedNumber);
StringBuilder sb = new StringBuilder("DataNode failed volumes:");
for (FsVolumeSpi vol : unhealthyVolumes) {
@@ -3424,8 +3424,8 @@ public class DataNode extends ReconfigurableBase
LOG.warn("Error occurred when removing unhealthy storage dirs", e);
}
LOG.debug("{}", sb);
- // send blockreport regarding volume failure
- handleDiskError(sb.toString());
+ // send blockreport regarding volume failure
+ handleDiskError(sb.toString(), failedNumber);
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 58a2f65..00590ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -370,9 +370,9 @@ public class DataNodeMetrics {
remoteBytesRead.incr(size);
}
}
-
- public void incrVolumeFailures() {
- volumeFailures.incr();
+
+ public void incrVolumeFailures(int size) {
+ volumeFailures.incr(size);
}
public void incrDatanodeNetworkErrors() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
index 4b4002b..2508eef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
@@ -77,6 +79,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
@@ -945,4 +948,27 @@ public class TestDataNodeVolumeFailure {
}
}, 10, 30 * 1000);
}
+
+ /*
+ * Fail two volumes, and check the metrics of VolumeFailures
+ */
+ @Test
+ public void testVolumeFailureTwo() throws Exception {
+ // fail two volumes
+ data_fail = cluster.getInstanceStorageDir(1, 0);
+ failedDir = MiniDFSCluster.getFinalizedDir(data_fail,
+ cluster.getNamesystem().getBlockPoolId());
+ failedDir.setReadOnly();
+ data_fail = cluster.getInstanceStorageDir(1, 1);
+ failedDir = MiniDFSCluster.getFinalizedDir(data_fail,
+ cluster.getNamesystem().getBlockPoolId());
+ failedDir.setReadOnly();
+
+ final DataNode dn = cluster.getDataNodes().get(1);
+ dn.checkDiskError();
+
+ MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
+ long volumeFailures = getLongCounter("VolumeFailures", rb);
+ assertEquals(2, volumeFailures);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-15219. DFS Client will stuck when
ResponseProcessor.run throw Error (#1902). Contributed by zhengchenyu.
Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d64f6881987085acb927248730bea26d5db959d3
Author: Isa Hekmatizadeh <es...@gmail.com>
AuthorDate: Tue Mar 24 17:47:22 2020 +0000
HDFS-15219. DFS Client will stuck when ResponseProcessor.run throw Error (#1902). Contributed by zhengchenyu.
(cherry picked from commit d9c4f1129c0814ab61fce6ea8baf4b272f84c252)
---
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 4c733bf..6637490 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1184,7 +1184,7 @@ class DataStreamer extends Daemon {
one.releaseBuffer(byteArrayManager);
}
- } catch (Exception e) {
+ } catch (Throwable e) {
if (!responderClosed) {
lastException.set(e);
errorState.setInternalError();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org