You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/02/23 19:46:52 UTC
hadoop git commit: HDFS-13162. Create Replica Trash directory on DN
startup. Contributed by Bharat Viswanadham.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-12996 59cf75887 -> 85a0ed79b
HDFS-13162. Create Replica Trash directory on DN startup. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85a0ed79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85a0ed79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85a0ed79
Branch: refs/heads/HDFS-12996
Commit: 85a0ed79b6640e9f367863bd82f801b5dc9b6cc1
Parents: 59cf758
Author: Hanisha Koneru <ha...@apache.org>
Authored: Fri Feb 23 11:46:25 2018 -0800
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Fri Feb 23 11:46:25 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 ++
.../hdfs/server/datanode/DataStorage.java | 5 +-
.../hdfs/server/datanode/StorageLocation.java | 12 +++++
.../src/main/resources/hdfs-default.xml | 10 ++++
.../hdfs/server/datanode/TestDataStorage.java | 55 +++++++++++++++++++-
5 files changed, 84 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85a0ed79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bea38d2..d99fac2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1079,6 +1079,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final boolean DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_DEFAULT =
false;
+ // Datanode Replica Trash
+ public static final String DFS_DATANODE_ENABLE_REPLICA_TRASH_KEY =
+ "dfs.datanode.enable.replica.trash";
+ public static final boolean DFS_DATANODE_ENABLE_REPLICA_TRASH_DEFAULT = false;
+
// Slow io warning log threshold settings for dfsclient and datanode.
public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.datanode.slow.io.warning.threshold.ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85a0ed79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index a85ae32..3e4d6f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -81,6 +81,7 @@ public class DataStorage extends Storage {
public final static String STORAGE_DIR_FINALIZED = "finalized";
public final static String STORAGE_DIR_LAZY_PERSIST = "lazypersist";
public final static String STORAGE_DIR_TMP = "tmp";
+ public final static String STORAGE_DIR_REPLICA_TRASH = "replica-trash";
/**
* Set of bpids for which 'trash' is currently enabled.
@@ -333,7 +334,7 @@ public class DataStorage extends Storage {
VolumeBuilder builder =
new VolumeBuilder(this, sd);
for (NamespaceInfo nsInfo : nsInfos) {
- location.makeBlockPoolDir(nsInfo.getBlockPoolID(), null);
+ location.makeBlockPoolDir(nsInfo.getBlockPoolID(), datanode.getConf());
final BlockPoolSliceStorage bpStorage = getBlockPoolSliceStorage(nsInfo);
final List<StorageDirectory> dirs = bpStorage.loadBpStorageDirectories(
@@ -448,7 +449,7 @@ public class DataStorage extends Storage {
final List<StorageDirectory> success = Lists.newArrayList();
final List<UpgradeTask> tasks = Lists.newArrayList();
for (StorageLocation dataDir : dataDirs) {
- dataDir.makeBlockPoolDir(bpid, null);
+ dataDir.makeBlockPoolDir(bpid, datanode.getConf());
try {
final List<Callable<StorageDirectory>> callables = Lists.newArrayList();
final List<StorageDirectory> dirs = bpStorage.recoverTransitionRead(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85a0ed79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
index 8ad51de..f3b5296 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java
@@ -198,6 +198,7 @@ public class StorageLocation
/**
* Create physical directory for block pools on the data node.
+ * If replica trash is enabled, create replica-trash folder.
*
* @param blockPoolID
* the block pool id
@@ -223,8 +224,19 @@ public class StorageLocation
DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY,
DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT));
File data = new File(getBpURI(blockPoolID, Storage.STORAGE_DIR_CURRENT));
+
+ boolean replicaTrashEnable = conf.getBoolean(
+ DFSConfigKeys.DFS_DATANODE_ENABLE_REPLICA_TRASH_KEY,
+ DFSConfigKeys.DFS_DATANODE_ENABLE_REPLICA_TRASH_DEFAULT);
+
try {
DiskChecker.checkDir(localFS, new Path(data.toURI()), permission);
+ if (replicaTrashEnable) {
+ File replicaTrash = new File(data,
+ DataStorage.STORAGE_DIR_REPLICA_TRASH);
+ DiskChecker.checkDir(localFS,
+ new Path(replicaTrash.toURI()), permission);
+ }
} catch (IOException e) {
DataStorage.LOG.warn("Invalid directory in: " + data.getCanonicalPath() +
": " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85a0ed79/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index d037b2a..7d1baa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3800,6 +3800,16 @@
</property>
<property>
+ <name>dfs.datanode.enable.replica.trash</name>
+ <value>false</value>
+ <description>
+ If true, datanode replica trash feature is enabled. This will move the
+ blocks invalidated to replica-trash folder under block pool directory,
+ which will help in the recovery of accidentally deleted data.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.balance.max.concurrent.moves</name>
<value>50</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85a0ed79/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
index 6c49451..95368be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -40,6 +41,7 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertFalse;
public class TestDataStorage {
private final static String DEFAULT_BPID = "bp-0";
@@ -51,12 +53,13 @@ public class TestDataStorage {
private final static StartupOption START_OPT = StartupOption.REGULAR;
private DataNode mockDN = Mockito.mock(DataNode.class);
+ private Configuration conf;
private NamespaceInfo nsInfo;
private DataStorage storage;
@Before
public void setUp() throws IOException {
- Configuration conf = new HdfsConfiguration();
+ conf = new HdfsConfiguration();
storage = new DataStorage();
nsInfo = new NamespaceInfo(0, CLUSTER_ID, DEFAULT_BPID, CTIME,
BUILD_VERSION, SOFTWARE_VERSION);
@@ -130,6 +133,56 @@ public class TestDataStorage {
assertTrue(bpSd.getVersionFile().isFile());
}
+
+ @Test
+ public void testReplicaTrashDirectory() throws IOException {
+ final int numLocations = 3;
+ final int numNamespace = 3;
+ List<StorageLocation> locations = createStorageLocations(numLocations);
+
+ // Add volumes for multiple namespaces.
+ List<NamespaceInfo> namespaceInfos = createNamespaceInfos(numNamespace);
+
+ conf.setBoolean(DFSConfigKeys.DFS_DATANODE_ENABLE_REPLICA_TRASH_KEY,
+ true);
+ mockDN.setConf(conf);
+
+ for (NamespaceInfo ni : namespaceInfos) {
+ storage.addStorageLocations(mockDN, ni, locations, START_OPT);
+ for (StorageLocation sl : locations) {
+ File currentDir = new File(new File(sl.getUri()),
+ Storage.STORAGE_DIR_CURRENT);
+ File bpDir = new File(currentDir, ni.getBlockPoolID());
+ File replicaTrashDir = new File(bpDir, DataStorage
+ .STORAGE_DIR_REPLICA_TRASH);
+ assertTrue(replicaTrashDir.isDirectory());
+ }
+ }
+ }
+
+ @Test
+ public void testNoReplicaTrashDirectory() throws IOException {
+ final int numLocations = 3;
+ final int numNamespace = 3;
+ List<StorageLocation> locations = createStorageLocations(numLocations);
+
+ // Add volumes for multiple namespaces.
+ List<NamespaceInfo> namespaceInfos = createNamespaceInfos(numNamespace);
+
+ for (NamespaceInfo ni : namespaceInfos) {
+ storage.addStorageLocations(mockDN, ni, locations, START_OPT);
+ for (StorageLocation sl : locations) {
+ File currentDir = new File(new File(sl.getUri()),
+ Storage.STORAGE_DIR_CURRENT);
+ File bpDir = new File(currentDir, ni.getBlockPoolID());
+ File replicaTrashDir = new File(bpDir, DataStorage
+ .STORAGE_DIR_REPLICA_TRASH);
+ assertFalse(replicaTrashDir.exists());
+ }
+ }
+ }
+
+
@Test
public void testAddStorageDirectories() throws IOException,
URISyntaxException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org