You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2021/07/01 05:07:09 UTC
[hadoop] branch trunk updated: HDFS-16086. Add volume information
to datanode log for tracing (#3136)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 56c7ada HDFS-16086. Add volume information to datanode log for tracing (#3136)
56c7ada is described below
commit 56c7ada7a524812208a2112412a757ec6be02e5e
Author: litao <to...@gmail.com>
AuthorDate: Thu Jul 1 13:06:56 2021 +0800
HDFS-16086. Add volume information to datanode log for tracing (#3136)
---
.../java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java | 6 ++++++
.../org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java | 9 +++++----
.../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 1 +
.../java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java | 7 ++++---
.../java/org/apache/hadoop/hdfs/server/datanode/Replica.java | 7 +++++++
.../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 3 ++-
.../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 5 +++++
.../apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 5 +++++
.../hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java | 6 ++++++
.../server/datanode/extdataset/ExternalReplicaInPipeline.java | 6 ++++++
10 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index d9baa8f..a092743 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.Replica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.protobuf.ByteString;
@@ -523,6 +524,7 @@ public abstract class BlockListAsLongs implements Iterable<BlockReportReplica> {
@InterfaceAudience.Private
public static class BlockReportReplica extends Block implements Replica {
private ReplicaState state;
+
private BlockReportReplica() {
}
public BlockReportReplica(Block block) {
@@ -557,6 +559,10 @@ public abstract class BlockListAsLongs implements Iterable<BlockReportReplica> {
throw new UnsupportedOperationException();
}
@Override
+ public FsVolumeSpi getVolume() {
+ throw new UnsupportedOperationException();
+ }
+ @Override
public boolean equals(Object o) {
return super.equals(o);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 52f25d7..248bc9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -1551,11 +1551,12 @@ class BlockReceiver implements Closeable {
DatanodeRegistration dnR = datanode.getDNRegistrationForBP(block
.getBlockPoolId());
ClientTraceLog.info(String.format(DN_CLIENTTRACE_FORMAT, inAddr,
- myAddr, block.getNumBytes(), "HDFS_WRITE", clientname, offset,
- dnR.getDatanodeUuid(), block, endTime - startTime));
+ myAddr, replicaInfo.getVolume(), block.getNumBytes(),
+ "HDFS_WRITE", clientname, offset, dnR.getDatanodeUuid(),
+ block, endTime - startTime));
} else {
- LOG.info("Received " + block + " size " + block.getNumBytes()
- + " from " + inAddr);
+ LOG.info("Received " + block + " on volume " + replicaInfo.getVolume()
+ + " size " + block.getNumBytes() + " from " + inAddr);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index c49825b..209db42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -268,6 +268,7 @@ public class DataNode extends ReconfigurableBase
public static final String DN_CLIENTTRACE_FORMAT =
"src: %s" + // src IP
", dest: %s" + // dst IP
+ ", volume: %s" + // volume
", bytes: %s" + // byte count
", op: %s" + // operation
", cliID: %s" + // DFSClient id
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 082259c..a80d0c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -587,7 +587,7 @@ class DataXceiver extends Receiver implements Runnable {
final String clientTraceFmt =
clientName.length() > 0 && ClientTraceLog.isInfoEnabled()
? String.format(DN_CLIENTTRACE_FORMAT, localAddress, remoteAddress,
- "%d", "HDFS_READ", clientName, "%d",
+ "", "%d", "HDFS_READ", clientName, "%d",
dnR.getDatanodeUuid(), block, "%d")
: dnR + " Served block " + block + " to " +
remoteAddress;
@@ -929,8 +929,9 @@ class DataXceiver extends Receiver implements Runnable {
if (isDatanode ||
stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) {
datanode.closeBlock(block, null, storageUuid, isOnTransientStorage);
- LOG.info("Received {} src: {} dest: {} of size {}",
- block, remoteAddress, localAddress, block.getNumBytes());
+ LOG.info("Received {} src: {} dest: {} volume: {} of size {}",
+ block, remoteAddress, localAddress, replica.getVolume(),
+ block.getNumBytes());
}
if(isClient) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
index b6e5ba9..42bd8b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
/**
* This represents block replicas which are stored in DataNode.
@@ -64,4 +65,10 @@ public interface Replica {
* Return true if the target volume is backed by RAM.
*/
public boolean isOnTransientStorage();
+
+ /**
+ * Get the volume of replica.
+ * @return the volume of replica
+ */
+ public FsVolumeSpi getVolume();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 706c078..7d5f33b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -232,7 +232,8 @@ class FsDatasetAsyncDiskService {
void deleteAsync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete,
ExtendedBlock block, String trashDirectory) {
LOG.info("Scheduling " + block.getLocalBlock()
- + " replica " + replicaToDelete + " for deletion");
+ + " replica " + replicaToDelete + " on volume " +
+ replicaToDelete.getVolume() + " for deletion");
ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask(
volumeRef, replicaToDelete, block, trashDirectory);
execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index fb5d11e..b399588 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1129,6 +1129,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
}
try {
+ LOG.debug("moving block {} from {} to {}", block,
+ replicaInfo.getVolume(), volumeRef.getVolume());
moveBlock(block, replicaInfo, volumeRef, useVolumeOnSameMount);
datanode.getMetrics().incrReplaceBlockOpOnSameHost();
if (useVolumeOnSameMount) {
@@ -1631,6 +1633,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
if (ref == null) {
ref = volumes.getNextVolume(storageType, storageId, b.getNumBytes());
}
+ LOG.debug("Creating Rbw, block: {} on volume: {}", b, ref.getVolume());
FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
// create an rbw file to hold block in the designated volume
@@ -1904,6 +1907,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ReplicaInPipeline newReplicaInfo;
try {
newReplicaInfo = v.createTemporary(b);
+ LOG.debug("creating temporary for block: {} on volume: {}",
+ b, ref.getVolume());
} catch (IOException e) {
IOUtils.cleanupWithLogger(null, ref);
throw e;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index b56fc81..73e8bf7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -416,6 +416,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
} while (deadLine > System.currentTimeMillis());
throw new IOException("Minimum length was not achieved within timeout");
}
+
+ @Override
+ public FsVolumeSpi getVolume() {
+ return getStorage(theBlock).getVolume();
+ }
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java
index 8ea7432..566c4bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplica.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode.extdataset;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.Replica;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
public class ExternalReplica implements Replica {
@@ -62,4 +63,9 @@ public class ExternalReplica implements Replica {
public boolean isOnTransientStorage() {
return false;
}
+
+ @Override
+ public FsVolumeSpi getVolume() {
+ return null;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
index b135411..084caf0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipeline;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.util.DataChecksum;
@@ -135,4 +136,9 @@ public class ExternalReplicaInPipeline implements ReplicaInPipeline {
public void waitForMinLength(long minLength, long time, TimeUnit unit)
throws IOException {
}
+
+ @Override
+ public FsVolumeSpi getVolume() {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org