You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2016/02/04 20:20:34 UTC
hadoop git commit: HDFS-9730. Storage ID update does not happen when
there is a layout change. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/trunk 22a2b2231 -> 496f33de0
HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/496f33de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/496f33de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/496f33de
Branch: refs/heads/trunk
Commit: 496f33de0ce80dc455cfd51f19612da6f9b914f9
Parents: 22a2b22
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Feb 4 13:19:54 2016 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Feb 4 13:19:54 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/DataStorage.java | 47 +++++++++-----------
2 files changed, 23 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/496f33de/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8e9fb0b..846eb2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2772,6 +2772,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric()
(Chang Li via vinayakumarb)
+ HDFS-9730. Storage ID update does not happen when there is a layout change
+ (Tsz Wo Nicholas Sze via kihwal)
+
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/496f33de/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 57bb8b2..7903194 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -518,11 +518,7 @@ public class DataStorage extends Storage {
this.cTime = 0;
setDatanodeUuid(datanodeUuid);
- if (sd.getStorageUuid() == null) {
- // Assign a new Storage UUID.
- sd.setStorageUuid(DatanodeStorage.generateUuid());
- }
-
+ createStorageID(sd, false);
writeProperties(sd);
}
@@ -696,7 +692,13 @@ public class DataStorage extends Storage {
// do upgrade
if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
- doUpgrade(sd, nsInfo, conf); // upgrade
+ if (federationSupported) {
+ // If the existing on-disk layout version supports federation,
+ // simply update the properties.
+ upgradeProperties(sd);
+ } else {
+ doUpgradePreFederation(sd, nsInfo, conf);
+ }
return true; // doUgrade already has written properties
}
@@ -710,7 +712,8 @@ public class DataStorage extends Storage {
}
/**
- * Upgrade -- Move current storage into a backup directory,
+ * Upgrade from a pre-federation layout.
+ * Move current storage into a backup directory,
* and hardlink all its blocks into the new current directory.
*
* Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23
@@ -729,25 +732,9 @@ public class DataStorage extends Storage {
* There should be only ONE namenode in the cluster for first
* time upgrade to 0.22
* @param sd storage directory
- * @throws IOException on error
*/
- void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo,
- final Configuration conf) throws IOException {
- // If the existing on-disk layout version supportes federation, simply
- // update its layout version.
- if (DataNodeLayoutVersion.supports(
- LayoutVersion.Feature.FEDERATION, layoutVersion)) {
- // The VERSION file is already read in. Override the layoutVersion
- // field and overwrite the file. The upgrade work is handled by
- // {@link BlockPoolSliceStorage#doUpgrade}
- LOG.info("Updating layout version from " + layoutVersion + " to "
- + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " for storage "
- + sd.getRoot());
- layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
- writeProperties(sd);
- return;
- }
-
+ void doUpgradePreFederation(final StorageDirectory sd,
+ final NamespaceInfo nsInfo, final Configuration conf) throws IOException {
final int oldLV = getLayoutVersion();
LOG.info("Upgrading storage directory " + sd.getRoot()
+ ".\n old LV = " + oldLV
@@ -791,15 +778,21 @@ public class DataStorage extends Storage {
linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf);
// 4. Write version file under <SD>/current
- layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
clusterID = nsInfo.getClusterID();
- writeProperties(sd);
+ upgradeProperties(sd);
// 5. Rename <SD>/previous.tmp to <SD>/previous
rename(tmpDir, prevDir);
LOG.info("Upgrade of " + sd.getRoot()+ " is complete");
+ }
+ void upgradeProperties(StorageDirectory sd) throws IOException {
createStorageID(sd, layoutVersion);
+ LOG.info("Updating layout version from " + layoutVersion
+ + " to " + HdfsServerConstants.DATANODE_LAYOUT_VERSION
+ + " for storage " + sd.getRoot());
+ layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
+ writeProperties(sd);
}
/**