You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/06/11 19:12:51 UTC
hadoop git commit: HDFS-8573. Move creation of restartMeta file logic
from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 95c73d49b -> b258b344b
HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b258b344
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b258b344
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b258b344
Branch: refs/heads/trunk
Commit: b258b344bb76af6492828201959e36b45f0f75b8
Parents: 95c73d4
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Jun 11 10:12:31 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Jun 11 10:12:31 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/BlockReceiver.java | 10 +---------
.../hdfs/server/datanode/ReplicaInPipeline.java | 15 ++++++++++++++-
.../server/datanode/ReplicaInPipelineInterface.java | 10 ++++++++++
.../hdfs/server/datanode/SimulatedFSDataset.java | 5 +++++
.../extdataset/ExternalReplicaInPipeline.java | 7 +++++++
6 files changed, 40 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3c85773..7b8370e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8549. Abort the balancer if an upgrade is in progress. (wang)
+ HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to
+ ReplicaInPipeline. (Eddy Xu via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 2e11600..c46892d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -23,7 +23,6 @@ import java.io.BufferedOutputStream;
import java.io.Closeable;
import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.FileDescriptor;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -839,15 +838,8 @@ class BlockReceiver implements Closeable {
// In case this datanode is shutting down for quick restart,
// send a special ack upstream.
if (datanode.isRestarting() && isClient && !isTransfer) {
- File blockFile = ((ReplicaInPipeline)replicaInfo).getBlockFile();
- File restartMeta = new File(blockFile.getParent() +
- File.pathSeparator + "." + blockFile.getName() + ".restart");
- if (restartMeta.exists() && !restartMeta.delete()) {
- LOG.warn("Failed to delete restart meta file: " +
- restartMeta.getPath());
- }
try (Writer out = new OutputStreamWriter(
- new FileOutputStream(restartMeta), "UTF-8")) {
+ replicaInfo.createRestartMetaStream(), "UTF-8")) {
// write out the current time.
out.write(Long.toString(Time.now() + restartBudget));
out.flush();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
index 0eb143a..d9406f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.RandomAccessFile;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -287,7 +288,19 @@ public class ReplicaInPipeline extends ReplicaInfo
throw e;
}
}
-
+
+ @Override
+ public OutputStream createRestartMetaStream() throws IOException {
+ File blockFile = getBlockFile();
+ File restartMeta = new File(blockFile.getParent() +
+ File.pathSeparator + "." + blockFile.getName() + ".restart");
+ if (restartMeta.exists() && !restartMeta.delete()) {
+ DataNode.LOG.warn("Failed to delete restart meta file: " +
+ restartMeta.getPath());
+ }
+ return new FileOutputStream(restartMeta);
+ }
+
@Override
public String toString() {
return super.toString()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java
index 0263d0f..ef9f3e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.IOException;
+import java.io.OutputStream;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.util.DataChecksum;
@@ -73,4 +74,13 @@ public interface ReplicaInPipelineInterface extends Replica {
*/
public ReplicaOutputStreams createStreams(boolean isCreate,
DataChecksum requestedChecksum) throws IOException;
+
+ /**
+ * Create an output stream to write restart metadata in case of datanode
+ * shutting down for quick restart.
+ *
+ * @return output stream for writing.
+ * @throws IOException if any error occurs
+ */
+ public OutputStream createRestartMetaStream() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 778dd28..6e5f07c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -263,6 +263,11 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override
+ public OutputStream createRestartMetaStream() throws IOException {
+ return new SimulatedOutputStream();
+ }
+
+ @Override
synchronized public long getBlockId() {
return theBlock.getBlockId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b258b344/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
index ad44500..a0039bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hdfs.server.datanode.extdataset;
import java.io.IOException;
+import java.io.OutputStream;
+
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
@@ -60,6 +62,11 @@ public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface {
}
@Override
+ public OutputStream createRestartMetaStream() throws IOException {
+ return null;
+ }
+
+ @Override
public long getBlockId() {
return 0;
}