You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2015/12/23 00:51:24 UTC
[25/50] [abbrv] hadoop git commit: HDFS-9533. seen_txid in the shared
edits directory is modified during bootstrapping. Contributed by Kihwal Lee.
HDFS-9533. seen_txid in the shared edits directory is modified during bootstrapping. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cb1e011
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cb1e011
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cb1e011
Branch: refs/heads/yarn-2877
Commit: 5cb1e0118b173a95c1f7bdfae1e58d7833d61c26
Parents: 03bab8d
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Dec 17 17:11:16 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Dec 17 17:11:16 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/NNStorage.java | 18 +++++++++++++++++-
.../hdfs/server/namenode/ha/BootstrapStandby.java | 6 ++++--
.../hdfs/server/namenode/FSImageTestUtil.java | 7 +++++++
.../server/namenode/ha/TestBootstrapStandby.java | 11 +++++++++++
5 files changed, 42 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fc9dfe9..f896544 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2540,6 +2540,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9516. Truncate file fails with data dirs on multiple disks.
(Plamen Jeliazkov via shv)
+ HDFS-9533. seen_txid in the shared edits directory is modified during
+ bootstrapping (kihwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index 9b63e72..ed36b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -482,8 +482,24 @@ public class NNStorage extends Storage implements Closeable,
* @param txid the txid that has been reached
*/
public void writeTransactionIdFileToStorage(long txid) {
+ writeTransactionIdFileToStorage(txid, null);
+ }
+
+ /**
+ * Write a small file in all available storage directories that
+ * indicates that the namespace has reached some given transaction ID.
+ *
+ * This is used when the image is loaded to avoid accidental rollbacks
+ * in the case where an edit log is fully deleted but there is no
+ * checkpoint. See TestNameEditsConfigs.testNameEditsConfigsFailure()
+ * @param txid the txid that has been reached
+ * @param type the type of directory
+ */
+ public void writeTransactionIdFileToStorage(long txid,
+ NameNodeDirType type) {
// Write txid marker in all storage directories
- for (StorageDirectory sd : storageDirs) {
+ for (Iterator<StorageDirectory> it = dirIterator(type); it.hasNext();) {
+ StorageDirectory sd = it.next();
try {
writeTransactionIdFile(sd, txid);
} catch(IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index f694a7f..d84b0fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.NNUpgradeUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
@@ -329,13 +330,14 @@ public class BootstrapStandby implements Tool, Configurable {
return ERR_CODE_LOGS_UNAVAILABLE;
}
- image.getStorage().writeTransactionIdFileToStorage(curTxId);
-
// Download that checkpoint into our storage directories.
MD5Hash hash = TransferFsImage.downloadImageToStorage(
proxyInfo.getHttpAddress(), imageTxId, storage, true, true);
image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
hash);
+
+ // Write seen_txid to the formatted image directories.
+ storage.writeTransactionIdFileToStorage(imageTxId, NameNodeDirType.IMAGE);
} catch (IOException ioe) {
throw ioe;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
index 68eff19..3884f87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
@@ -568,4 +568,11 @@ public abstract class FSImageTestUtil {
FSImageTestUtil.assertParallelFilesAreIdentical(curDirs,
ignoredFiles);
}
+
+ public static long getStorageTxId(NameNode node, URI storageUri)
+ throws IOException {
+ StorageDirectory sDir = getFSImage(node).getStorage().
+ getStorageDirectory(storageUri);
+ return NNStorage.readTransactionIdFile(sDir);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cb1e011/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
index 8c39548..48d505c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
@@ -146,6 +146,13 @@ public class TestBootstrapStandby {
.getFSImage().getMostRecentCheckpointTxId();
assertEquals(6, expectedCheckpointTxId);
+ // advance the current txid
+ cluster.getFileSystem(0).create(new Path("/test_txid"), (short)1).close();
+
+ // obtain the content of seen_txid
+ URI editsUri = cluster.getSharedEditsDir(0, maxNNCount - 1);
+ long seen_txid_shared = FSImageTestUtil.getStorageTxId(nn0, editsUri);
+
for (int i = 1; i < maxNNCount; i++) {
assertEquals(0, forceBootstrap(i));
@@ -156,6 +163,10 @@ public class TestBootstrapStandby {
}
FSImageTestUtil.assertNNFilesMatch(cluster);
+ // Make sure the seen_txid was not modified by the standby
+ assertEquals(seen_txid_shared,
+ FSImageTestUtil.getStorageTxId(nn0, editsUri));
+
// We should now be able to start the standby successfully.
restartNameNodesFromIndex(1);
}